vendor/llvm/llvm-trunk-r321017

author: Dimitry Andric <dim@FreeBSD.org> 2017-12-18 20:10:56 +0000
committer: Dimitry Andric <dim@FreeBSD.org> 2017-12-18 20:10:56 +0000
commit: 044eb2f6afba375a914ac9d8024f8f5142bb912e (patch)
tree: 1475247dc9f9fe5be155ebd4c9069c75aadf8c20 /test/CodeGen/AMDGPU
parent: eb70dddbd77e120e5d490bd8fbe7ff3f8fa81c6b (diff)
458 files changed, 25342 insertions, 4256 deletions
diff --git a/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll b/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
index cdfb667c26bd7..8f83feac51d86 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
+++ b/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
@@ -5,7 +5,7 @@
 
 ; Tests for add.
 ; CHECK: name: addi32
-; CHECK: {{%[0-9]+}}(s32) = G_ADD
+; CHECK: {{%[0-9]+}}:_(s32) = G_ADD
 define amdgpu_kernel void @addi32(i32 %arg1, i32 %arg2) {
   %res = add i32 %arg1, %arg2
   store i32 %res, i32 addrspace(1)* undef
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
index 56a9e7022db9c..9b53b029691eb 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
@@ -13,7 +13,7 @@ legalized:       true
 regBankSelected: true
 
 # GCN: global_addrspace
-# GCN: [[PTR:%[0-9]+]] = COPY %vgpr0_vgpr1
+# GCN: [[PTR:%[0-9]+]]:vreg_64 = COPY %vgpr0_vgpr1
 # GCN: FLAT_LOAD_DWORD  [[PTR]], 0, 0, 0
 
 body: |
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir
index ea2ad2ba83a52..70e2b5e4ae2b8 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-smrd.mir
@@ -14,83 +14,83 @@ legalized:       true
 regBankSelected: true
 
 # GCN: body:
-# GCN: [[PTR:%[0-9]+]] = COPY %sgpr0_sgpr1
+# GCN: [[PTR:%[0-9]+]]:sreg_64 = COPY %sgpr0_sgpr1
 
 # Immediate offset:
 # SICI: S_LOAD_DWORD_IMM [[PTR]], 1, 0
-# VI:   S_LOAD_DWORD_IMM [[PTR]], 4, 0 
+# VI:   S_LOAD_DWORD_IMM [[PTR]], 4, 0
 
 # Max immediate offset for SI
 # SICI: S_LOAD_DWORD_IMM [[PTR]], 255, 0
 # VI:   S_LOAD_DWORD_IMM [[PTR]], 1020, 0
 
 # Immediate overflow for SI
-# SI: [[K1024:%[0-9]+]] = S_MOV_B32 1024
+# SI: [[K1024:%[0-9]+]]:sreg_32 = S_MOV_B32 1024
 # SI: S_LOAD_DWORD_SGPR [[PTR]], [[K1024]], 0
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 256, 0
 # VI: S_LOAD_DWORD_IMM [[PTR]], 1024, 0
 
 # Max immediate offset for VI
-# SI: [[K1048572:%[0-9]+]] = S_MOV_B32 1048572
+# SI: [[K1048572:%[0-9]+]]:sreg_32 = S_MOV_B32 1048572
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 262143
 # VI: S_LOAD_DWORD_IMM [[PTR]], 1048572
 
 #
 # Immediate overflow for VI
-# SIVI: [[K1048576:%[0-9]+]] = S_MOV_B32 1048576
+# SIVI: [[K1048576:%[0-9]+]]:sreg_32 = S_MOV_B32 1048576
 # SIVI: S_LOAD_DWORD_SGPR [[PTR]], [[K1048576]], 0
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 262144, 0
 
 # Max immediate for CI
-# SIVI: [[K_LO:%[0-9]+]] = S_MOV_B32 4294967292
-# SIVI: [[K_HI:%[0-9]+]] = S_MOV_B32 3
-# SIVI: [[K:%[0-9]+]] = REG_SEQUENCE [[K_LO]], 1, [[K_HI]], 2
-# SIVI-DAG: [[K_SUB0:%[0-9]+]] = COPY [[K]].sub0
-# SIVI-DAG: [[PTR_LO:%[0-9]+]] = COPY [[PTR]].sub0
-# SIVI: [[ADD_PTR_LO:%[0-9]+]] = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
-# SIVI-DAG: [[K_SUB1:%[0-9]+]] = COPY [[K]].sub1
-# SIVI-DAG: [[PTR_HI:%[0-9]+]] = COPY [[PTR]].sub1
-# SIVI: [[ADD_PTR_HI:%[0-9]+]] = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
-# SIVI: [[ADD_PTR:%[0-9]+]] = REG_SEQUENCE [[ADD_PTR_LO]], 1, [[ADD_PTR_HI]], 2
+# SIVI: [[K_LO:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967292
+# SIVI: [[K_HI:%[0-9]+]]:sreg_32 = S_MOV_B32 3
+# SIVI: [[K:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[K_LO]], %subreg.sub0, [[K_HI]], %subreg.sub1
+# SIVI-DAG: [[K_SUB0:%[0-9]+]]:sgpr_32 = COPY [[K]].sub0
+# SIVI-DAG: [[PTR_LO:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub0
+# SIVI: [[ADD_PTR_LO:%[0-9]+]]:sreg_32 = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
+# SIVI-DAG: [[K_SUB1:%[0-9]+]]:sgpr_32 = COPY [[K]].sub1
+# SIVI-DAG: [[PTR_HI:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub1
+# SIVI: [[ADD_PTR_HI:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
+# SIVI: [[ADD_PTR:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[ADD_PTR_LO]], %subreg.sub0, [[ADD_PTR_HI]], %subreg.sub1
 # SIVI: S_LOAD_DWORD_IMM [[ADD_PTR]], 0, 0
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 4294967295, 0
 
 # Immediate overflow for CI
-# GCN: [[K_LO:%[0-9]+]] = S_MOV_B32 0
-# GCN: [[K_HI:%[0-9]+]] = S_MOV_B32 4
-# GCN: [[K:%[0-9]+]] = REG_SEQUENCE [[K_LO]], 1, [[K_HI]], 2
-# GCN-DAG: [[K_SUB0:%[0-9]+]] = COPY [[K]].sub0
-# GCN-DAG: [[PTR_LO:%[0-9]+]] = COPY [[PTR]].sub0
-# GCN: [[ADD_PTR_LO:%[0-9]+]] = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
-# GCN-DAG: [[K_SUB1:%[0-9]+]] = COPY [[K]].sub1
-# GCN-DAG: [[PTR_HI:%[0-9]+]] = COPY [[PTR]].sub1
-# GCN: [[ADD_PTR_HI:%[0-9]+]] = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
-# GCN: [[ADD_PTR:%[0-9]+]] = REG_SEQUENCE [[ADD_PTR_LO]], 1, [[ADD_PTR_HI]], 2
+# GCN: [[K_LO:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+# GCN: [[K_HI:%[0-9]+]]:sreg_32 = S_MOV_B32 4
+# GCN: [[K:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[K_LO]], %subreg.sub0, [[K_HI]], %subreg.sub1
+# GCN-DAG: [[K_SUB0:%[0-9]+]]:sgpr_32 = COPY [[K]].sub0
+# GCN-DAG: [[PTR_LO:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub0
+# GCN: [[ADD_PTR_LO:%[0-9]+]]:sreg_32 = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
+# GCN-DAG: [[K_SUB1:%[0-9]+]]:sgpr_32 = COPY [[K]].sub1
+# GCN-DAG: [[PTR_HI:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub1
+# GCN: [[ADD_PTR_HI:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
+# GCN: [[ADD_PTR:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[ADD_PTR_LO]], %subreg.sub0, [[ADD_PTR_HI]], %subreg.sub1
 # GCN: S_LOAD_DWORD_IMM [[ADD_PTR]], 0, 0
 
 # Max 32-bit byte offset
-# SIVI: [[K4294967292:%[0-9]+]] = S_MOV_B32 4294967292
+# SIVI: [[K4294967292:%[0-9]+]]:sreg_32 = S_MOV_B32 4294967292
 # SIVI: S_LOAD_DWORD_SGPR [[PTR]], [[K4294967292]], 0
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 1073741823, 0
 
 # Overflow 32-bit byte offset
-# SIVI: [[K_LO:%[0-9]+]] = S_MOV_B32 0
-# SIVI: [[K_HI:%[0-9]+]] = S_MOV_B32 1
-# SIVI: [[K:%[0-9]+]] = REG_SEQUENCE [[K_LO]], 1, [[K_HI]], 2
-# SIVI-DAG: [[K_SUB0:%[0-9]+]] = COPY [[K]].sub0
-# SIVI-DAG: [[PTR_LO:%[0-9]+]] = COPY [[PTR]].sub0
-# SIVI: [[ADD_PTR_LO:%[0-9]+]] = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
-# SIVI-DAG: [[K_SUB1:%[0-9]+]] = COPY [[K]].sub1
-# SIVI-DAG: [[PTR_HI:%[0-9]+]] = COPY [[PTR]].sub1
-# SIVI: [[ADD_PTR_HI:%[0-9]+]] = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
-# SIVI: [[ADD_PTR:%[0-9]+]] = REG_SEQUENCE [[ADD_PTR_LO]], 1, [[ADD_PTR_HI]], 2
+# SIVI: [[K_LO:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+# SIVI: [[K_HI:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+# SIVI: [[K:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[K_LO]], %subreg.sub0, [[K_HI]], %subreg.sub1
+# SIVI-DAG: [[K_SUB0:%[0-9]+]]:sgpr_32 = COPY [[K]].sub0
+# SIVI-DAG: [[PTR_LO:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub0
+# SIVI: [[ADD_PTR_LO:%[0-9]+]]:sreg_32 = S_ADD_U32 [[PTR_LO]], [[K_SUB0]]
+# SIVI-DAG: [[K_SUB1:%[0-9]+]]:sgpr_32 = COPY [[K]].sub1
+# SIVI-DAG: [[PTR_HI:%[0-9]+]]:sgpr_32 = COPY [[PTR]].sub1
+# SIVI: [[ADD_PTR_HI:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[PTR_HI]], [[K_SUB1]]
+# SIVI: [[ADD_PTR:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[ADD_PTR_LO]], %subreg.sub0, [[ADD_PTR_HI]], %subreg.sub1
 # SIVI: S_LOAD_DWORD_IMM [[ADD_PTR]], 0, 0
 # CI: S_LOAD_DWORD_IMM_ci [[PTR]], 1073741824, 0
 
 body: |
   bb.0:
     liveins: %sgpr0_sgpr1
-   
+
     %0:sgpr(p2) = COPY %sgpr0_sgpr1
 
     %1:sgpr(s64) = G_CONSTANT i64 4
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
index ea435725bf25d..0b8092778bd49 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
@@ -13,8 +13,8 @@ legalized:       true
 regBankSelected: true
 
 # GCN: global_addrspace
-# GCN: [[PTR:%[0-9]+]] = COPY %vgpr0_vgpr1
-# GCN: [[VAL:%[0-9]+]] = COPY %vgpr2
+# GCN: [[PTR:%[0-9]+]]:vreg_64 = COPY %vgpr0_vgpr1
+# GCN: [[VAL:%[0-9]+]]:vgpr_32 = COPY %vgpr2
 # GCN: FLAT_STORE_DWORD [[PTR]], [[VAL]], 0, 0, 0
 
 body: |
diff --git a/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
new file mode 100644
index 0000000000000..ebcdac39274a4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -0,0 +1,60 @@
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
+
+
+; CHECK-LABEL: name: test_f32_inreg
+; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY %sgpr0
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]]
+define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
+  call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
+  ret void
+}
+
+; CHECK-LABEL: name: test_f32
+; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY %vgpr0
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]]
+define amdgpu_vs void @test_f32(float %arg0) {
+  call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
+  ret void
+}
+
+; CHECK-LABEL: name: test_ptr2_byval
+; CHECK: [[S01:%[0-9]+]]:_(p2) = COPY %sgpr0_sgpr1
+; CHECK: G_LOAD [[S01]]
+define amdgpu_vs void @test_ptr2_byval(i32 addrspace(2)* byval %arg0) {
+   %tmp0 = load volatile i32, i32 addrspace(2)* %arg0
+  ret void
+}
+
+; CHECK-LABEL: name: test_ptr2_inreg
+; CHECK: [[S01:%[0-9]+]]:_(p2) = COPY %sgpr0_sgpr1
+; CHECK: G_LOAD [[S01]]
+define amdgpu_vs void @test_ptr2_inreg(i32 addrspace(2)* inreg %arg0) {
+  %tmp0 = load volatile i32, i32 addrspace(2)* %arg0
+  ret void
+}
+
+; CHECK-LABEL: name: test_sgpr_alignment0
+; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY %sgpr0
+; CHECK: [[S23:%[0-9]+]]:_(p2) = COPY %sgpr2_sgpr3
+; CHECK: G_LOAD [[S23]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]]
+define amdgpu_vs void @test_sgpr_alignment0(float inreg %arg0, i32 addrspace(2)* inreg %arg1) {
+  %tmp0 = load volatile i32, i32 addrspace(2)* %arg1
+  call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
+  ret void
+}
+
+; CHECK-LABEL: name: test_order
+; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY %sgpr0
+; CHECK: [[S1:%[0-9]+]]:_(s32) = COPY %sgpr1
+; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY %vgpr0
+; CHECK: [[V1:%[0-9]+]]:_(s32) = COPY %vgpr1
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]](s32), [[S0]](s32), [[V1]](s32), [[S1]](s32)
+define amdgpu_vs void @test_order(float inreg %arg0, float inreg %arg1, float %arg2, float %arg3) {
+  call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg2, float %arg0, float %arg3, float %arg1, i1 false, i1 false) #0
+  ret void
+}
+
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)  #0
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
index f10c896a7af66..3afb2c7db5e60 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -13,10 +14,13 @@ registers:
 body: |
   bb.0:
     liveins: %vgpr0, %vgpr1
-    ; CHECK-LABEL: name: test_add
-    ; CHECK: %2(s32) = G_ADD %0, %1
 
+    ; CHECK-LABEL: name: test_add
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %vgpr1
+    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
     %0(s32) = COPY %vgpr0
     %1(s32) = COPY %vgpr1
     %2(s32) = G_ADD %0, %1
+    %vgpr0 = COPY %2
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index 50ef150510d22..c22355cffe35d 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -13,10 +14,13 @@ registers:
 body: |
   bb.0:
     liveins: %vgpr0, %vgpr1
-    ; CHECK-LABEL: name: test_and
-    ; CHECK: %2(s32) = G_AND %0, %1
 
+    ; CHECK-LABEL: name: test_and
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %vgpr1
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[COPY1]]
     %0(s32) = COPY %vgpr0
     %1(s32) = COPY %vgpr1
     %2(s32) = G_AND %0, %1
+    %vgpr0 = COPY %2
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
index e27c313b8ec0f..1f0fbd0a36894 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -13,11 +14,13 @@ registers:
 body: |
   bb.0:
     liveins: %vgpr0
-    ; CHECK-LABEL: name: test_bitcast
-    ; CHECK: %1(<2 x s16>) = G_BITCAST %0
-    ; CHECK: %2(s32) = G_BITCAST %1
 
+    ; CHECK-LABEL: name: test_bitcast
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](s32)
+    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
     %0(s32) = COPY %vgpr0
     %1(<2 x s16>) = G_BITCAST %0
     %2(s32) = G_BITCAST %1
+    %vgpr0 = COPY %2
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
index b3e41c7751c53..843c03f6cc2ca 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -24,10 +25,11 @@ registers:
   - { id: 1, class: _ }
 body: |
   bb.0.entry:
-    ; CHECK-LABEL: name: test_constant
-    ; CHECK: %0(s32) = G_CONSTANT i32 5
-    ; CHECK: %1(s1) = G_CONSTANT i1 false
 
+    ; CHECK-LABEL: name: test_constant
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32), [[C1]](s1), [[C1]](s1)
     %0(s32) = G_CONSTANT i32 5
     %1(s1) = G_CONSTANT i1 0
     G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), %0, %0, %0, %0, %0, %0, %1, %1;
@@ -40,10 +42,12 @@ registers:
   - { id: 1, class: _ }
 body: |
   bb.0.entry:
-    ; CHECK-LABEL: name: test_fconstant
-    ; CHECK: %0(s32) = G_FCONSTANT  float 1.000000e+00
-    ; CHECK: %1(s32) = G_FCONSTANT  float 7.5
 
+    ; CHECK-LABEL: name: test_fconstant
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 7.500000e+00
     %0(s32) = G_FCONSTANT float 1.0
+    %vgpr0 = COPY %0
     %1(s32) = G_FCONSTANT float 7.5
+    %vgpr0 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
new file mode 100644
index 0000000000000..0db061ea8a9c9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -0,0 +1,27 @@
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+
+--- |
+  define void @test_fadd() {
+  entry:
+    ret void
+  }
+
+...
+
+---
+name:            test_fadd
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body: |
+  bb.0.entry:
+    liveins: %vgpr0, %vgpr1
+    ; CHECK-LABEL: name: test_fadd
+    ; CHECK: %2:_(s32) = G_FADD %0, %1
+
+    %0(s32) = COPY %vgpr0
+    %1(s32) = COPY %vgpr1
+    %2(s32) = G_FADD %0, %1
+    %vgpr0 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
new file mode 100644
index 0000000000000..98bbd6a2af307
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -0,0 +1,26 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+
+--- |
+  define void @test_fmul() { ret void }
+...
+
+---
+name:            test_fmul
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body: |
+  bb.0:
+    liveins: %vgpr0, %vgpr1
+
+    ; CHECK-LABEL: name: test_fmul
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %vgpr1
+    ; CHECK: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
+    %0(s32) = COPY %vgpr0
+    %1(s32) = COPY %vgpr1
+    %2(s32) = G_FMUL %0, %1
+    %vgpr0 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
index ebd473d769b37..bb6b68ad2360c 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -march=amdgcn -mcpu=fiji  -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -16,9 +17,15 @@ registers:
 body: |
   bb.0.entry:
     liveins: %vgpr0
+    ; CHECK-LABEL: name: test_icmp
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C]], [[COPY]]
+    ; CHECK: %vgpr0 = COPY [[SELECT]](s32)
     %0(s32) = G_CONSTANT i32 0
     %1(s32) = COPY %vgpr0
-
-    ; CHECK: %2(s1) = G_ICMP intpred(ne), %0(s32), %1
     %2(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SELECT %2(s1), %0(s32), %1(s32)
+    %vgpr0 = COPY %3
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
new file mode 100644
index 0000000000000..8780a1e080a20
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -0,0 +1,25 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+
+--- |
+  define void @test_or() { ret void }
+...
+---
+name:            test_or
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body: |
+  bb.0:
+    liveins: %vgpr0, %vgpr1
+
+    ; CHECK-LABEL: name: test_or
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %vgpr1
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[COPY1]]
+    %0(s32) = COPY %vgpr0
+    %1(s32) = COPY %vgpr1
+    %2(s32) = G_OR %0, %1
+    %vgpr0 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index d11130936bd9b..af633789182c1 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -march=amdgcn -mcpu=fiji  -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 --- |
@@ -16,13 +17,20 @@ registers:
 body: |
   bb.0:
     liveins: %vgpr0
+    ; CHECK-LABEL: name: test_select
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C1]], [[C2]]
     %0(s32) = G_CONSTANT i32 0
     %1(s32) = COPY %vgpr0
 
     %2(s1) = G_ICMP intpred(ne), %0, %1
     %3(s32) = G_CONSTANT i32 1
     %4(s32) = G_CONSTANT i32 2
-    ; CHECK: %5(s32) = G_SELECT %2(s1), %3, %4
     %5(s32) = G_SELECT %2, %3, %4
+    %vgpr0 = COPY %5
 
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index 3d5251d102072..d6320fa3e69a2 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 ---
@@ -9,10 +10,13 @@ registers:
 body: |
   bb.0.entry:
     liveins: %vgpr0, %vgpr1
-    ; CHECK-LABEL: name: test_shl
-    ; CHECK: %2(s32) = G_SHL %0, %1
 
+    ; CHECK-LABEL: name: test_shl
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %vgpr1
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[COPY1]]
     %0(s32) = COPY %vgpr0
     %1(s32) = COPY %vgpr1
     %2(s32) = G_SHL %0, %1
+    %vgpr0 = COPY %2
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/smrd.ll b/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
index 8a6b3df9cff8d..6629188131daf 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
+++ b/test/CodeGen/AMDGPU/GlobalISel/smrd.ll
@@ -1,5 +1,5 @@
 ; FIXME: Need to add support for mubuf stores to enable this on SI.
-; XUN: llc < %s -march=amdgcn -mcpu=SI -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=SI --check-prefix=GCN --check-prefix=SIVI %s
+; XUN: llc < %s -march=amdgcn -mcpu=tahiti -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=SI --check-prefix=GCN --check-prefix=SIVI %s
 ; RUN: llc < %s -march=amdgcn -mcpu=bonaire -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=CI --check-prefix=GCN %s
 ; RUN: llc < %s -march=amdgcn -mcpu=tonga -show-mc-encoding -verify-machineinstrs -global-isel | FileCheck --check-prefix=VI --check-prefix=GCN --check-prefix=SIVI %s
 
diff --git a/test/CodeGen/AMDGPU/InlineAsmCrash.ll b/test/CodeGen/AMDGPU/InlineAsmCrash.ll
new file mode 100644
index 0000000000000..8ad1cbb9a32f5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/InlineAsmCrash.ll
@@ -0,0 +1,12 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK: ;;#ASMSTART
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: ;;#ASMEND
+
+define void @foo(i32* %ptr) {
+  %tmp = tail call { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } asm "s_nop 0", "=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,=v,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65"(i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2)
+  %tmp2 = extractvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %tmp, 0
+  store i32 %tmp2, i32* %ptr, align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/add.i16.ll b/test/CodeGen/AMDGPU/add.i16.ll
index 98848295a73b2..8da3401b3dbe2 100644
--- a/test/CodeGen/AMDGPU/add.i16.ll
+++ b/test/CodeGen/AMDGPU/add.i16.ll
@@ -105,7 +105,7 @@ define amdgpu_kernel void @v_test_add_i16_zext_to_i64(i64 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i32:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]],  [[A]], [[B]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]],  [[B]], [[A]]
 ; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: buffer_store_dword [[SEXT]]
 define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@@ -125,7 +125,7 @@ define amdgpu_kernel void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i64:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
+; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
 ; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 ; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
diff --git a/test/CodeGen/AMDGPU/add.ll b/test/CodeGen/AMDGPU/add.ll
index 6dcd7c234dc6d..ec3bf6dca57f3 100644
--- a/test/CodeGen/AMDGPU/add.ll
+++ b/test/CodeGen/AMDGPU/add.ll
@@ -1,14 +1,15 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-;FUNC-LABEL: {{^}}test1:
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; FUNC-LABEL: {{^}}s_add_i32:
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-;SI: s_add_i32 s[[REG:[0-9]+]], {{s[0-9]+, s[0-9]+}}
-;SI: v_mov_b32_e32 v[[REG]], s[[REG]]
-;SI: buffer_store_dword v[[REG]],
-define amdgpu_kernel void @test1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+; GCN: s_add_i32 s[[REG:[0-9]+]], {{s[0-9]+, s[0-9]+}}
+; GCN: v_mov_b32_e32 v[[REG]], s[[REG]]
+; GCN: buffer_store_dword v[[REG]],
+define amdgpu_kernel void @s_add_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %a = load i32, i32 addrspace(1)* %in
   %b = load i32, i32 addrspace(1)* %b_ptr
@@ -17,14 +18,13 @@ define amdgpu_kernel void @test1(i32 addrspace(1)* %out, i32 addrspace(1)* %in)
   ret void
 }
 
-;FUNC-LABEL: {{^}}test2:
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; FUNC-LABEL: {{^}}s_add_v2i32:
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
-
-define amdgpu_kernel void @test2(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+define amdgpu_kernel void @s_add_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %in
   %b = load <2 x i32>, <2 x i32> addrspace(1)* %b_ptr
@@ -33,18 +33,17 @@ define amdgpu_kernel void @test2(<2 x i32> addrspace(1)* %out, <2 x i32> addrspa
   ret void
 }
 
-;FUNC-LABEL: {{^}}test4:
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-;EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
-
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
-;SI: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+; FUNC-LABEL: {{^}}s_add_v4i32:
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
+; EG: ADD_INT {{[* ]*}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-define amdgpu_kernel void @test4(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+; GCN: s_add_i32 s{{[0-9]+, s[0-9]+, s[0-9]+}}
+define amdgpu_kernel void @s_add_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
   %a = load <4 x i32>, <4 x i32> addrspace(1)* %in
   %b = load <4 x i32>, <4 x i32> addrspace(1)* %b_ptr
@@ -53,7 +52,7 @@ define amdgpu_kernel void @test4(<4 x i32> addrspace(1)* %out, <4 x i32> addrspa
   ret void
 }
 
-; FUNC-LABEL: {{^}}test8:
+; FUNC-LABEL: {{^}}s_add_v8i32:
 ; EG: ADD_INT
 ; EG: ADD_INT
 ; EG: ADD_INT
@@ -63,22 +62,22 @@ define amdgpu_kernel void @test4(<4 x i32> addrspace(1)* %out, <4 x i32> addrspa
 ; EG: ADD_INT
 ; EG: ADD_INT
 
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-define amdgpu_kernel void @test8(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> %b) {
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+define amdgpu_kernel void @s_add_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> %b) {
 entry:
   %0 = add <8 x i32> %a, %b
   store <8 x i32> %0, <8 x i32> addrspace(1)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}test16:
+; FUNC-LABEL: {{^}}s_add_v16i32:
 ; EG: ADD_INT
 ; EG: ADD_INT
 ; EG: ADD_INT
@@ -96,32 +95,62 @@ entry:
 ; EG: ADD_INT
 ; EG: ADD_INT
 
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-; SI: s_add_i32
-define amdgpu_kernel void @test16(<16 x i32> addrspace(1)* %out, <16 x i32> %a, <16 x i32> %b) {
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+; GCN: s_add_i32
+define amdgpu_kernel void @s_add_v16i32(<16 x i32> addrspace(1)* %out, <16 x i32> %a, <16 x i32> %b) {
 entry:
   %0 = add <16 x i32> %a, %b
   store <16 x i32> %0, <16 x i32> addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}v_add_i32:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc, [[B]], [[A]]
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, [[A]], [[B]]
+define amdgpu_kernel void @v_add_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %gep, i32 1
+  %a = load volatile i32, i32 addrspace(1)* %gep
+  %b = load volatile i32, i32 addrspace(1)* %b_ptr
+  %result = add i32 %a, %b
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_add_imm_i32:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc, 0x7b, [[A]]
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 0x7b, [[A]]
+define amdgpu_kernel void @v_add_imm_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %gep, i32 1
+  %a = load volatile i32, i32 addrspace(1)* %gep
+  %result = add i32 %a, 123
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}add64:
-; SI: s_add_u32
-; SI: s_addc_u32
+; GCN: s_add_u32
+; GCN: s_addc_u32
 
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.XY]]
 ; EG-DAG: ADD_INT {{[* ]*}}
@@ -131,8 +160,8 @@ entry:
 ; EG-NOT: SUB
 define amdgpu_kernel void @add64(i64 addrspace(1)* %out, i64 %a, i64 %b) {
 entry:
-  %0 = add i64 %a, %b
-  store i64 %0, i64 addrspace(1)* %out
+  %add = add i64 %a, %b
+  store i64 %add, i64 addrspace(1)* %out
   ret void
 }
 
@@ -142,7 +171,7 @@ entry:
 ; to a VGPR before doing the add.
 
 ; FUNC-LABEL: {{^}}add64_sgpr_vgpr:
-; SI-NOT: v_addc_u32_e32 s
+; GCN-NOT: v_addc_u32_e32 s
 
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.XY]]
 ; EG-DAG: ADD_INT {{[* ]*}}
@@ -160,8 +189,8 @@ entry:
 
 ; Test i64 add inside a branch.
 ; FUNC-LABEL: {{^}}add64_in_branch:
-; SI: s_add_u32
-; SI: s_addc_u32
+; GCN: s_add_u32
+; GCN: s_addc_u32
 
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[LO:T[0-9]+\.XY]]
 ; EG-DAG: ADD_INT {{[* ]*}}
@@ -187,3 +216,8 @@ endif:
   store i64 %3, i64 addrspace(1)* %out
   ret void
 }
+
+declare i32 @llvm.r600.read.tidig.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/add.v2i16.ll b/test/CodeGen/AMDGPU/add.v2i16.ll
index 4baa35ca57c58..a6c2901bd422f 100644
--- a/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
@@ -52,8 +52,8 @@ define amdgpu_kernel void @s_test_add_self_v2i16(<2 x i16> addrspace(1)* %out, <
 ; GCN-LABEL: {{^}}s_test_add_v2i16_kernarg:
 ; GFX9: v_pk_add_u16 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 
-; VI: v_add_i32
-; VI: v_add_i32_sdwa
+; VI: v_add_u32
+; VI: v_add_u32_sdwa
 define amdgpu_kernel void @s_test_add_v2i16_kernarg(<2 x i16> addrspace(1)* %out, <2 x i16> %a, <2 x i16> %b) #1 {
   %add = add <2 x i16> %a, %b
   store <2 x i16> %add, <2 x i16> addrspace(1)* %out
@@ -155,18 +155,18 @@ define amdgpu_kernel void @v_test_add_v2i16_inline_fp_split(<2 x i16> addrspace(
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_add_v2i16_zext_to_v2i32:
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_and_b32_e32 v[[ELT0:[0-9]+]], 0xffff, [[ADD]]
 ; GFX9-DAG: v_lshrrev_b32_e32 v[[ELT1:[0-9]+]], 16, [[ADD]]
 ; GFX9: buffer_store_dwordx2 v{{\[}}[[ELT0]]:[[ELT1]]{{\]}}
 
-; VI: flat_load_ushort v[[A_HI:[0-9]+]]
 ; VI: flat_load_ushort v[[A_LO:[0-9]+]]
-; VI: flat_load_ushort v[[B_HI:[0-9]+]]
+; VI: flat_load_ushort v[[A_HI:[0-9]+]]
 ; VI: flat_load_ushort v[[B_LO:[0-9]+]]
+; VI: flat_load_ushort v[[B_HI:[0-9]+]]
 
 ; VI: v_add_u16_e32 v[[ADD_HI:[0-9]+]], v[[A_HI]], v[[B_HI]]
 ; VI-NOT: and
@@ -190,8 +190,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i32(<2 x i32> addrspace(1)
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_add_v2i16_zext_to_v2i64:
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_and_b32_e32 v[[ELT0:[0-9]+]], 0xffff, [[ADD]]
@@ -223,8 +223,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(<2 x i64> addrspace(1)
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_add_v2i16_sext_to_v2i32:
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_bfe_i32 v[[ELT0:[0-9]+]], [[ADD]], 0, 16
@@ -251,8 +251,8 @@ define amdgpu_kernel void @v_test_add_v2i16_sext_to_v2i32(<2 x i32> addrspace(1)
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_add_v2i16_sext_to_v2i64:
-; GCN: flat_load_dword
-; GCN: flat_load_dword
+; GCN: {{flat|global}}_load_dword
+; GCN: {{flat|global}}_load_dword
 
 ; GFX9: v_pk_add_u16
 ; GFX9: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/addrspacecast.ll b/test/CodeGen/AMDGPU/addrspacecast.ll
index a6aa9e7951515..901b197b1b8f8 100644
--- a/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=HSA -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=HSA -check-prefix=CI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=HSA -check-prefix=GFX9 %s
 
 ; HSA-LABEL: {{^}}use_group_to_flat_addrspacecast:
@@ -147,7 +147,7 @@ define amdgpu_kernel void @use_flat_to_private_addrspacecast(i32 addrspace(4)* %
 ; HSA-DAG: v_mov_b32_e32 v[[VPTRLO:[0-9]+]], s[[PTRLO]]
 ; HSA-DAG: v_mov_b32_e32 v[[VPTRHI:[0-9]+]], s[[PTRHI]]
 ; HSA-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0
-; HSA: flat_store_dword v{{\[}}[[VPTRLO]]:[[VPTRHI]]{{\]}}, [[K]]
+; HSA: {{flat|global}}_store_dword v{{\[}}[[VPTRLO]]:[[VPTRHI]]{{\]}}, [[K]]
 define amdgpu_kernel void @use_flat_to_global_addrspacecast(i32 addrspace(4)* %ptr) #0 {
   %ftos = addrspacecast i32 addrspace(4)* %ptr to i32 addrspace(1)*
   store volatile i32 0, i32 addrspace(1)* %ftos
@@ -176,7 +176,7 @@ define amdgpu_kernel void @use_flat_to_constant_addrspacecast(i32 addrspace(4)*
 
 ; HSA-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0{{$}}
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 7{{$}}
-; HSA: flat_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
+; HSA: {{flat|global}}_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
 define amdgpu_kernel void @cast_0_group_to_flat_addrspacecast() #0 {
   %cast = addrspacecast i32 addrspace(3)* null to i32 addrspace(4)*
   store volatile i32 7, i32 addrspace(4)* %cast
@@ -197,7 +197,7 @@ define amdgpu_kernel void @cast_0_flat_to_group_addrspacecast() #0 {
 ; HSA: v_mov_b32_e32 v[[LO:[0-9]+]], 0{{$}}
 ; HSA: v_mov_b32_e32 v[[K:[0-9]+]], 7{{$}}
 ; HSA: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
-; HSA: flat_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
+; HSA: {{flat|global}}_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
 define amdgpu_kernel void @cast_neg1_group_to_flat_addrspacecast() #0 {
   %cast = addrspacecast i32 addrspace(3)* inttoptr (i32 -1 to i32 addrspace(3)*) to i32 addrspace(4)*
   store volatile i32 7, i32 addrspace(4)* %cast
@@ -222,7 +222,7 @@ define amdgpu_kernel void @cast_neg1_flat_to_group_addrspacecast() #0 {
 ; HSA-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0{{$}}
 ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 7{{$}}
 ; HSA: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
-; HSA: flat_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
+; HSA: {{flat|global}}_store_dword v{{\[}}[[LO]]:[[HI]]{{\]}}, v[[K]]
 define amdgpu_kernel void @cast_0_private_to_flat_addrspacecast() #0 {
   %cast = addrspacecast i32* null to i32 addrspace(4)*
   store volatile i32 7, i32 addrspace(4)* %cast
@@ -242,7 +242,7 @@ define amdgpu_kernel void @cast_0_flat_to_private_addrspacecast() #0 {
 ; specialize away generic pointer accesses.
 
 ; HSA-LABEL: {{^}}branch_use_flat_i32:
-; HSA: flat_store_dword {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}
+; HSA: {{flat|global}}_store_dword {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}
 ; HSA: s_endpgm
 define amdgpu_kernel void @branch_use_flat_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* %gptr, i32 addrspace(3)* %lptr, i32 %x, i32 %c) #0 {
 entry:
@@ -274,9 +274,9 @@ end:
 ; GFX9: s_add_u32 flat_scratch_lo, s6, s9
 ; GFX9: s_addc_u32 flat_scratch_hi, s7, 0
 
-; HSA: flat_store_dword
+; HSA: {{flat|global}}_store_dword
 ; HSA: s_barrier
-; HSA: flat_load_dword
+; HSA: {{flat|global}}_load_dword
 define amdgpu_kernel void @store_flat_scratch(i32 addrspace(1)* noalias %out, i32) #0 {
   %alloca = alloca i32, i32 9, align 4
   %x = call i32 @llvm.amdgcn.workitem.id.x() #2
diff --git a/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll b/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll
new file mode 100644
index 0000000000000..e967723384bf7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/adjust-writemask-invalid-copy.ll
@@ -0,0 +1,84 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}adjust_writemask_crash_0_nochain:
+; GCN: image_get_lod v0, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} dmask:0x2
+; GCN-NOT: v1
+; GCN-NOT: v0
+; GCN: buffer_store_dword v0
+define amdgpu_ps void @adjust_writemask_crash_0_nochain() #0 {
+main_body:
+  %tmp = call <2 x float> @llvm.amdgcn.image.getlod.v2f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 false, i1 false, i1 false, i1 false, i1 false)
+  %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
+  %tmp4 = extractelement <4 x float> %tmp3, i32 0
+  store volatile float %tmp4, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}adjust_writemask_crash_1_nochain:
+; GCN: image_get_lod v0, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} dmask:0x1
+; GCN-NOT: v1
+; GCN-NOT: v0
+; GCN: buffer_store_dword v0
+define amdgpu_ps void @adjust_writemask_crash_1_nochain() #0 {
+main_body:
+  %tmp = call <2 x float> @llvm.amdgcn.image.getlod.v2f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 false, i1 false, i1 false, i1 false, i1 false)
+  %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
+  %tmp4 = extractelement <4 x float> %tmp3, i32 1
+  store volatile float %tmp4, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}adjust_writemask_crash_0_chain:
+; GCN: image_sample v0, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} dmask:0x2
+; GCN-NOT: v1
+; GCN-NOT: v0
+; GCN: buffer_store_dword v0
+define amdgpu_ps void @adjust_writemask_crash_0_chain() #0 {
+main_body:
+  %tmp = call <2 x float> @llvm.amdgcn.image.sample.v2f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 false, i1 false, i1 false, i1 false, i1 false)
+  %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
+  %tmp4 = extractelement <4 x float> %tmp3, i32 0
+  store volatile float %tmp4, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}adjust_writemask_crash_1_chain:
+; GCN: image_sample v0, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} dmask:0x1
+; GCN-NOT: v1
+; GCN-NOT: v0
+; GCN: buffer_store_dword v0
+define amdgpu_ps void @adjust_writemask_crash_1_chain() #0 {
+main_body:
+  %tmp = call <2 x float> @llvm.amdgcn.image.sample.v2f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 false, i1 false, i1 false, i1 false, i1 false)
+  %tmp1 = bitcast <2 x float> %tmp to <2 x i32>
+  %tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+  %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
+  %tmp4 = extractelement <4 x float> %tmp3, i32 1
+  store volatile float %tmp4, float addrspace(1)* undef
+  ret void
+}
+
+define amdgpu_ps void @adjust_writemask_crash_0_v4() #0 {
+main_body:
+  %tmp = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 5, i1 false, i1 false, i1 false, i1 false, i1 false)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %tmp3 = bitcast <4 x i32> %tmp2 to <4 x float>
+  %tmp4 = extractelement <4 x float> %tmp3, i32 0
+  store volatile float %tmp4, float addrspace(1)* undef
+  ret void
+}
+
+
+declare <2 x float> @llvm.amdgcn.image.sample.v2f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.image.getlod.v2f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.image.getlod.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
diff --git a/test/CodeGen/AMDGPU/amdgcn.private-memory.ll b/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
index 79450b97c218a..1d5c538427b97 100644
--- a/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
+++ b/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
@@ -13,7 +13,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
 ; GCN-LABEL: {{^}}work_item_info:
 ; GCN-NOT: v0
-; GCN: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, v0, v{{[0-9]+}}
+; GCN: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, v0, v{{[0-9]+}}
 ; GCN: buffer_store_dword [[RESULT]]
 define amdgpu_kernel void @work_item_info(i32 addrspace(1)* %out, i32 %in) {
 entry:
diff --git a/test/CodeGen/AMDGPU/amdgpu-inline.ll b/test/CodeGen/AMDGPU/amdgpu-inline.ll
new file mode 100644
index 0000000000000..8af4b3348b1f4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdgpu-inline.ll
@@ -0,0 +1,152 @@
+; RUN: opt -mtriple=amdgcn--amdhsa -O3 -S -amdgpu-function-calls -inline-threshold=1 < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-INL1 %s
+; RUN: opt -mtriple=amdgcn--amdhsa -O3 -S -amdgpu-function-calls < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-INLDEF %s
+
+define coldcc float @foo(float %x, float %y) {
+entry:
+  %cmp = fcmp ogt float %x, 0.000000e+00
+  %div = fdiv float %y, %x
+  %mul = fmul float %x, %y
+  %cond = select i1 %cmp, float %div, float %mul
+  ret float %cond
+}
+
+define coldcc void @foo_private_ptr(float* nocapture %p) {
+entry:
+  %tmp1 = load float, float* %p, align 4
+  %cmp = fcmp ogt float %tmp1, 1.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %div = fdiv float 1.000000e+00, %tmp1
+  store float %div, float* %p, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define coldcc void @foo_private_ptr2(float* nocapture %p1, float* nocapture %p2) {
+entry:
+  %tmp1 = load float, float* %p1, align 4
+  %cmp = fcmp ogt float %tmp1, 1.000000e+00
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %div = fdiv float 2.000000e+00, %tmp1
+  store float %div, float* %p2, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define coldcc float @sin_wrapper(float %x) {
+bb:
+  %call = tail call float @_Z3sinf(float %x)
+  ret float %call
+}
+
+define void @foo_noinline(float* nocapture %p) #0 {
+entry:
+  %tmp1 = load float, float* %p, align 4
+  %mul = fmul float %tmp1, 2.000000e+00
+  store float %mul, float* %p, align 4
+  ret void
+}
+
+; GCN: define amdgpu_kernel void @test_inliner(
+; GCN-INL1:   %c1 = tail call coldcc float @foo(
+; GCN-INLDEF: %cmp.i = fcmp ogt float %tmp2, 0.000000e+00
+; GCN:        %div.i{{[0-9]*}} = fdiv float 1.000000e+00, %c
+; GCN:        %div.i{{[0-9]*}} = fdiv float 2.000000e+00, %tmp1.i
+; GCN:        call void @foo_noinline(
+; GCN:        tail call float @_Z3sinf(
+define amdgpu_kernel void @test_inliner(float addrspace(1)* nocapture %a, i32 %n) {
+entry:
+  %pvt_arr = alloca [64 x float], align 4
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i32 %tid
+  %tmp2 = load float, float addrspace(1)* %arrayidx, align 4
+  %add = add i32 %tid, 1
+  %arrayidx2 = getelementptr inbounds float, float addrspace(1)* %a, i32 %add
+  %tmp5 = load float, float addrspace(1)* %arrayidx2, align 4
+  %c1 = tail call coldcc float @foo(float %tmp2, float %tmp5)
+  %or = or i32 %tid, %n
+  %arrayidx5 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 %or
+  store float %c1, float* %arrayidx5, align 4
+  %arrayidx7 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 %or
+  call coldcc void @foo_private_ptr(float* %arrayidx7)
+  %arrayidx8 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 1
+  %arrayidx9 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 2
+  call coldcc void @foo_private_ptr2(float* %arrayidx8, float* %arrayidx9)
+  call void @foo_noinline(float* %arrayidx7)
+  %and = and i32 %tid, %n
+  %arrayidx11 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 %and
+  %tmp12 = load float, float* %arrayidx11, align 4
+  %c2 = call coldcc float @sin_wrapper(float %tmp12)
+  store float %c2, float* %arrayidx7, align 4
+  %xor = xor i32 %tid, %n
+  %arrayidx16 = getelementptr inbounds [64 x float], [64 x float]* %pvt_arr, i32 0, i32 %xor
+  %tmp16 = load float, float* %arrayidx16, align 4
+  store float %tmp16, float addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+; GCN: define amdgpu_kernel void @test_inliner_multi_pvt_ptr(
+; GCN: %div.i{{[0-9]*}} = fdiv float 2.000000e+00, %tmp1.i
+define amdgpu_kernel void @test_inliner_multi_pvt_ptr(float addrspace(1)* nocapture %a, i32 %n, float %v) {
+entry:
+  %pvt_arr1 = alloca [32 x float], align 4
+  %pvt_arr2 = alloca [32 x float], align 4
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i32 %tid
+  %or = or i32 %tid, %n
+  %arrayidx4 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 %or
+  %arrayidx5 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr2, i32 0, i32 %or
+  store float %v, float* %arrayidx4, align 4
+  store float %v, float* %arrayidx5, align 4
+  %arrayidx8 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 1
+  %arrayidx9 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr2, i32 0, i32 2
+  call coldcc void @foo_private_ptr2(float* %arrayidx8, float* %arrayidx9)
+  %xor = xor i32 %tid, %n
+  %arrayidx15 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 %xor
+  %arrayidx16 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr2, i32 0, i32 %xor
+  %tmp15 = load float, float* %arrayidx15, align 4
+  %tmp16 = load float, float* %arrayidx16, align 4
+  %tmp17 = fadd float %tmp15, %tmp16
+  store float %tmp17, float addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+; GCN: define amdgpu_kernel void @test_inliner_multi_pvt_ptr_cutoff(
+; GCN-INL1:   call coldcc void @foo_private_ptr2
+; GCN-INLDEF: %div.i{{[0-9]*}} = fdiv float 2.000000e+00, %tmp1.i
+define amdgpu_kernel void @test_inliner_multi_pvt_ptr_cutoff(float addrspace(1)* nocapture %a, i32 %n, float %v) {
+entry:
+  %pvt_arr1 = alloca [32 x float], align 4
+  %pvt_arr2 = alloca [33 x float], align 4
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i32 %tid
+  %or = or i32 %tid, %n
+  %arrayidx4 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 %or
+  %arrayidx5 = getelementptr inbounds [33 x float], [33 x float]* %pvt_arr2, i32 0, i32 %or
+  store float %v, float* %arrayidx4, align 4
+  store float %v, float* %arrayidx5, align 4
+  %arrayidx8 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 1
+  %arrayidx9 = getelementptr inbounds [33 x float], [33 x float]* %pvt_arr2, i32 0, i32 2
+  call coldcc void @foo_private_ptr2(float* %arrayidx8, float* %arrayidx9)
+  %xor = xor i32 %tid, %n
+  %arrayidx15 = getelementptr inbounds [32 x float], [32 x float]* %pvt_arr1, i32 0, i32 %xor
+  %arrayidx16 = getelementptr inbounds [33 x float], [33 x float]* %pvt_arr2, i32 0, i32 %xor
+  %tmp15 = load float, float* %arrayidx15, align 4
+  %tmp16 = load float, float* %arrayidx16, align 4
+  %tmp17 = fadd float %tmp15, %tmp16
+  store float %tmp17, float addrspace(1)* %arrayidx, align 4
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+declare float @_Z3sinf(float) #1
+
+attributes #0 = { noinline }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/amdgpu-shader-calling-convention.ll b/test/CodeGen/AMDGPU/amdgpu-shader-calling-convention.ll
index 0ba8836b20dca..4993b95d228e2 100644
--- a/test/CodeGen/AMDGPU/amdgpu-shader-calling-convention.ll
+++ b/test/CodeGen/AMDGPU/amdgpu-shader-calling-convention.ll
@@ -3,7 +3,7 @@
 
 
 ; GCN-LABEL: {{^}}shader_cc:
-; GCN: v_add_i32_e32 v0, vcc, s8, v0
+; GCN: v_add_{{[iu]}}32_e32 v0, vcc, s8, v0
 define amdgpu_cs float @shader_cc(<4 x i32> inreg, <4 x i32> inreg, i32 inreg %w, float %v) {
   %vi = bitcast float %v to i32
   %x = add i32 %vi, %w
diff --git a/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
index 1f4b1eaa209a0..228d3c7d43067 100644
--- a/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
+++ b/test/CodeGen/AMDGPU/amdgpu.private-memory.ll
@@ -1,12 +1,12 @@
-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck -enable-var-scope -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck -enable-var-scope -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE %s
 ; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA
-; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC
+; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck -enable-var-scope -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA %s
+; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck -enable-var-scope -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -enable-amdgpu-aa=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck -enable-var-scope -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s
 
-; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=HSAOPT -check-prefix=OPT %s
-; RUN: opt -S -mtriple=amdgcn-unknown-unknown -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=NOHSAOPT -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -enable-var-scope -check-prefix=HSAOPT -check-prefix=OPT %s
+; RUN: opt -S -mtriple=amdgcn-unknown-unknown -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -enable-var-scope -check-prefix=NOHSAOPT -check-prefix=OPT %s
 
 ; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck %s -check-prefix=R600 -check-prefix=FUNC
 
@@ -251,8 +251,7 @@ entry:
 
 ; R600: MOVA_INT
 
-; SI-PROMOTE-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4 ; encoding:
-; SI-PROMOTE-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:5 ; encoding:
+; SI-PROMOTE-DAG: buffer_store_short v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4 ; encoding:
 
 ; SI-ALLOCA-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4 ; encoding: [0x04,0x00,0x60,0xe0
 ; SI-ALLOCA-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:5 ; encoding: [0x05,0x00,0x60,0xe0
@@ -391,7 +390,8 @@ entry:
 ; FUNC-LABEL: ptrtoint:
 ; SI-NOT: ds_write
 ; SI: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen
-; SI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:5 ;
+; SI: v_add_{{[iu]}}32_e32 [[ADD_OFFSET:v[0-9]+]], vcc, 5,
+; SI: buffer_load_dword v{{[0-9]+}}, [[ADD_OFFSET:v[0-9]+]], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen ;
 define amdgpu_kernel void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
   %alloca = alloca [16 x i32]
   %tmp0 = getelementptr [16 x i32], [16 x i32]* %alloca, i32 0, i32 %a
diff --git a/test/CodeGen/AMDGPU/amdpal-cs.ll b/test/CodeGen/AMDGPU/amdpal-cs.ll
new file mode 100644
index 0000000000000..6ffca4c4565e6
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-cs.ll
@@ -0,0 +1,12 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; amdpal compute shader: check for 0x2e12 (COMPUTE_PGM_RSRC1) in pal metadata
+; GCN-LABEL: {{^}}cs_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2e12,
+define amdgpu_cs half @cs_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
diff --git a/test/CodeGen/AMDGPU/amdpal-es.ll b/test/CodeGen/AMDGPU/amdpal-es.ll
new file mode 100644
index 0000000000000..049e9ae2564e3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-es.ll
@@ -0,0 +1,12 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+; amdpal evaluation shader: check for 0x2cca (SPI_SHADER_PGM_RSRC1_ES) in pal metadata
+; GCN-LABEL: {{^}}es_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2cca,
+define amdgpu_es half @es_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+
diff --git a/test/CodeGen/AMDGPU/amdpal-gs.ll b/test/CodeGen/AMDGPU/amdpal-gs.ll
new file mode 100644
index 0000000000000..89e7cf92c1f65
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-gs.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; amdpal geometry shader: check for 0x2c8a (SPI_SHADER_PGM_RSRC1_GS) in pal metadata
+; GCN-LABEL: {{^}}gs_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2c8a,
+define amdgpu_gs half @gs_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+
diff --git a/test/CodeGen/AMDGPU/amdpal-hs.ll b/test/CodeGen/AMDGPU/amdpal-hs.ll
new file mode 100644
index 0000000000000..b867717ee2377
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-hs.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; amdpal hull shader: check for 0x2d0a (SPI_SHADER_PGM_RSRC1_HS) in pal metadata
+; GCN-LABEL: {{^}}hs_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2d0a,
+define amdgpu_hs half @hs_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+
diff --git a/test/CodeGen/AMDGPU/amdpal-ls.ll b/test/CodeGen/AMDGPU/amdpal-ls.ll
new file mode 100644
index 0000000000000..7168a3c77b87b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-ls.ll
@@ -0,0 +1,12 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+
+; amdpal load shader: check for 0x2d4a (SPI_SHADER_PGM_RSRC1_LS) in pal metadata
+; GCN-LABEL: {{^}}ls_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2d4a,
+define amdgpu_ls half @ls_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+
diff --git a/test/CodeGen/AMDGPU/amdpal-ps.ll b/test/CodeGen/AMDGPU/amdpal-ps.ll
new file mode 100644
index 0000000000000..5e19c774a761e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-ps.ll
@@ -0,0 +1,17 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; amdpal pixel shader: check for 0x2c0a (SPI_SHADER_PGM_RSRC1_PS) in pal
+; metadata. Check for 0x2c0b (SPI_SHADER_PGM_RSRC2_PS) in pal metadata, and
+; it has a value starting 0x42 as it is set to 0x42000000 in the metadata
+; below. Also check that key 0x10000000 value 0x12345678 is propagated.
+; GCN-LABEL: {{^}}ps_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*0x2c0a,[^,]*,0x2c0b,0x42.*,0x10000000,0x12345678}}
+define amdgpu_ps half @ps_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+!amdgpu.pal.metadata = !{!0}
+!0 = !{i32 11275, i32 1107296256, i32 268435456, i32 305419896}
diff --git a/test/CodeGen/AMDGPU/amdpal-psenable.ll b/test/CodeGen/AMDGPU/amdpal-psenable.ll
new file mode 100644
index 0000000000000..c1494d0d4c43b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-psenable.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; This pixel shader does not use the result of its interpolation, so it would
+; end up with an interpolation mode set in PSAddr but not PSEnable. This test tests
+; the workaround that ensures that an interpolation mode is also set in PSEnable.
+; GCN-LABEL: {{^}}amdpal_psenable:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0xa1b3,0x2,0xa1b4,0x2,
+define amdgpu_ps void @amdpal_psenable(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <2 x float> %pos) #6 {
+  %inst23 = extractelement <2 x float> %pos, i32 0
+  %inst24 = extractelement <2 x float> %pos, i32 1
+  %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)
+  %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)
+  ret void
+}
+
+declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2
+declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2
+
+attributes #6 = { nounwind "InitialPSInputAddr"="2" }
diff --git a/test/CodeGen/AMDGPU/amdpal-vs.ll b/test/CodeGen/AMDGPU/amdpal-vs.ll
new file mode 100644
index 0000000000000..a13205dabd045
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal-vs.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+
+; amdpal vertex shader: check for 45352 (SPI_SHADER_PGM_RSRC1_VS) in pal metadata
+; GCN-LABEL: {{^}}vs_amdpal:
+; GCN: .amd_amdgpu_pal_metadata{{.*}}0x2c4a,
+define amdgpu_vs half @vs_amdpal(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+
diff --git a/test/CodeGen/AMDGPU/amdpal.ll b/test/CodeGen/AMDGPU/amdpal.ll
new file mode 100644
index 0000000000000..baa56de919e4c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/amdpal.ll
@@ -0,0 +1,58 @@
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=tahiti | FileCheck --check-prefix=PAL --enable-var-scope %s
+
+; PAL: .AMDGPU.config
+
+define amdgpu_kernel void @simple(i32 addrspace(1)* %out) {
+entry:
+  store i32 0, i32 addrspace(1)* %out
+  ret void
+}
+
+; Check code sequence for amdpal use of scratch for alloca. This is the case
+; where the high half of the address comes from s_getpc.
+
+; PAL-LABEL: {{^}}scratch:
+; PAL: s_getpc_b64 s{{\[}}[[GITPTR:[0-9]+]]:
+; PAL: s_mov_b32 s[[GITPTR]], s0
+; PAL: s_load_dwordx4 s{{\[}}[[SCRATCHDESC:[0-9]+]]:{{[0-9]+]}}, s{{\[}}[[GITPTR]]:
+; PAL: buffer_store{{.*}}, s{{\[}}[[SCRATCHDESC]]:
+
+define amdgpu_kernel void @scratch(<2 x i32> %in, i32 %idx, i32* %out) {
+entry:
+  %v = alloca [2 x i32]
+  %vv = bitcast [2 x i32]* %v to <2 x i32>*
+  store <2 x i32> %in, <2 x i32>* %vv
+  %e = getelementptr [2 x i32], [2 x i32]* %v, i32 0, i32 %idx
+  %x = load i32, i32* %e
+  store i32 %x, i32* %out
+  ret void
+}
+
+; Check code sequence for amdpal use of scratch for alloca. This is the case
+; where the amdgpu-git-ptr-high function attribute gives the high half of the
+; address to use.
+; Looks like you can't do arithmetic on a filecheck variable, so we can't test
+; that the s_movk_i32 is into a reg that is one more than the following
+; s_mov_b32.
+
+; PAL-LABEL: {{^}}scratch2:
+; PAL: s_movk_i32 s{{[0-9]+}}, 0x1234
+; PAL: s_mov_b32 s[[GITPTR:[0-9]+]], s0
+; PAL: s_load_dwordx4 s{{\[}}[[SCRATCHDESC:[0-9]+]]:{{[0-9]+]}}, s{{\[}}[[GITPTR]]:
+; PAL: buffer_store{{.*}}, s{{\[}}[[SCRATCHDESC]]:
+
+define amdgpu_kernel void @scratch2(<2 x i32> %in, i32 %idx, i32* %out) #0 {
+entry:
+  %v = alloca [2 x i32]
+  %vv = bitcast [2 x i32]* %v to <2 x i32>*
+  store <2 x i32> %in, <2 x i32>* %vv
+  %e = getelementptr [2 x i32], [2 x i32]* %v, i32 0, i32 %idx
+  %x = load i32, i32* %e
+  store i32 %x, i32* %out
+  ret void
+}
+
+attributes #0 = { nounwind "amdgpu-git-ptr-high"="0x1234" }
+
+; Check we have CS_NUM_USED_VGPRS in PAL metadata.
+; PAL: .amd_amdgpu_pal_metadata {{.*}},0x10000027,
diff --git a/test/CodeGen/AMDGPU/and.ll b/test/CodeGen/AMDGPU/and.ll
index ee0190149e92e..c320e0609c288 100644
--- a/test/CodeGen/AMDGPU/and.ll
+++ b/test/CodeGen/AMDGPU/and.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() #0
@@ -83,7 +83,8 @@ define amdgpu_kernel void @s_and_multi_use_constant_i32_0(i32 addrspace(1)* %out
 ; SI: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, [[K]]
 ; SI: s_add_i32
 ; SI: s_add_i32 [[ADD:s[0-9]+]], s{{[0-9]+}}, [[K]]
-; SI: buffer_store_dword [[VK]]
+; SI: v_mov_b32_e32 [[VADD:v[0-9]+]], [[ADD]]
+; SI: buffer_store_dword [[VADD]]
 define amdgpu_kernel void @s_and_multi_use_constant_i32_1(i32 addrspace(1)* %out, i32 %a, i32 %b) {
   %and = and i32 %a, 1234567
   %foo = add i32 %and, 1234567
@@ -219,10 +220,10 @@ define amdgpu_kernel void @s_and_32_bit_constant_i64(i64 addrspace(1)* %out, i64
 }
 
 ; FUNC-LABEL: {{^}}s_and_multi_use_inline_imm_i64:
+; SI: s_load_dwordx2
 ; SI: s_load_dword [[A:s[0-9]+]]
 ; SI: s_load_dword [[B:s[0-9]+]]
 ; SI: s_load_dwordx2
-; SI: s_load_dwordx2
 ; SI-NOT: and
 ; SI: s_lshl_b32 [[A]], [[A]], 1
 ; SI: s_lshl_b32 [[B]], [[B]], 1
diff --git a/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll b/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
index e9797eff712b7..51cd6c43e03a5 100644
--- a/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
+++ b/test/CodeGen/AMDGPU/annotate-kernel-features-hsa-call.ll
@@ -237,52 +237,59 @@ define void @func_indirect_use_kernarg_segment_ptr() #1 {
   ret void
 }
 
-; HSA: define void @use_implicitarg_ptr() #14 {
+; HSA: define amdgpu_kernel void @kern_use_implicitarg_ptr() #15 {
+define amdgpu_kernel void @kern_use_implicitarg_ptr() #1 {
+  %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  store volatile i8 addrspace(2)* %implicitarg.ptr, i8 addrspace(2)* addrspace(1)* undef
+  ret void
+}
+
+; HSA: define void @use_implicitarg_ptr() #15 {
 define void @use_implicitarg_ptr() #1 {
   %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
   store volatile i8 addrspace(2)* %implicitarg.ptr, i8 addrspace(2)* addrspace(1)* undef
   ret void
 }
 
-; HSA: define void @func_indirect_use_implicitarg_ptr() #14 {
+; HSA: define void @func_indirect_use_implicitarg_ptr() #15 {
 define void @func_indirect_use_implicitarg_ptr() #1 {
   call void @use_implicitarg_ptr()
   ret void
 }
 
-; HSA: declare void @external.func() #15
+; HSA: declare void @external.func() #16
 declare void @external.func() #3
 
-; HSA: define internal void @defined.func() #15 {
+; HSA: define internal void @defined.func() #16 {
 define internal void @defined.func() #3 {
   ret void
 }
 
-; HSA: define void @func_call_external() #15 {
+; HSA: define void @func_call_external() #16 {
 define void @func_call_external() #3 {
   call void @external.func()
   ret void
 }
 
-; HSA: define void @func_call_defined() #15 {
+; HSA: define void @func_call_defined() #16 {
 define void @func_call_defined() #3 {
   call void @defined.func()
   ret void
 }
 
-; HSA: define void @func_call_asm() #15 {
+; HSA: define void @func_call_asm() #16 {
 define void @func_call_asm() #3 {
   call void asm sideeffect "", ""() #3
   ret void
 }
 
-; HSA: define amdgpu_kernel void @kern_call_external() #16 {
+; HSA: define amdgpu_kernel void @kern_call_external() #17 {
 define amdgpu_kernel void @kern_call_external() #3 {
   call void @external.func()
   ret void
 }
 
-; HSA: define amdgpu_kernel void @func_kern_defined() #16 {
+; HSA: define amdgpu_kernel void @func_kern_defined() #17 {
 define amdgpu_kernel void @func_kern_defined() #3 {
   call void @defined.func()
   ret void
@@ -308,5 +315,6 @@ attributes #3 = { nounwind }
 ; HSA: attributes #12 = { nounwind "target-cpu"="gfx900" }
 ; HSA: attributes #13 = { nounwind "amdgpu-queue-ptr" "target-cpu"="gfx900" }
 ; HSA: attributes #14 = { nounwind "amdgpu-kernarg-segment-ptr" "target-cpu"="fiji" }
-; HSA: attributes #15 = { nounwind }
-; HSA: attributes #16 = { nounwind "amdgpu-flat-scratch" }
+; HSA: attributes #15 = { nounwind "amdgpu-implicitarg-ptr" "target-cpu"="fiji" }
+; HSA: attributes #16 = { nounwind }
+; HSA: attributes #17 = { nounwind "amdgpu-flat-scratch" }
diff --git a/test/CodeGen/AMDGPU/anyext.ll b/test/CodeGen/AMDGPU/anyext.ll
index 3f220c4084129..cabf0578aaf0c 100644
--- a/test/CodeGen/AMDGPU/anyext.ll
+++ b/test/CodeGen/AMDGPU/anyext.ll
@@ -1,12 +1,13 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
 
 ; GCN-LABEL: {{^}}anyext_i1_i32:
 ; GCN: v_cndmask_b32_e64
-define amdgpu_kernel void @anyext_i1_i32(i32 addrspace(1)* %out, i32 %cond) {
+define amdgpu_kernel void @anyext_i1_i32(i32 addrspace(1)* %out, i32 %cond) #0 {
 entry:
   %tmp = icmp eq i32 %cond, 0
   %tmp1 = zext i1 %tmp to i8
@@ -18,11 +19,11 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}s_anyext_i16_i32:
-; VI: v_add_u16_e32 [[ADD:v[0-9]+]],
-; VI: v_xor_b32_e32 [[XOR:v[0-9]+]], -1, [[ADD]]
-; VI: v_and_b32_e32 [[AND:v[0-9]+]], 1, [[XOR]]
-; VI: buffer_store_dword [[AND]]
-define amdgpu_kernel void @s_anyext_i16_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %a, i16 addrspace(1)* %b) {
+; GFX89: v_add_u16_e32 [[ADD:v[0-9]+]],
+; GFX89: v_xor_b32_e32 [[XOR:v[0-9]+]], -1, [[ADD]]
+; GFX89: v_and_b32_e32 [[AND:v[0-9]+]], 1, [[XOR]]
+; GFX89: buffer_store_dword [[AND]]
+define amdgpu_kernel void @s_anyext_i16_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %a, i16 addrspace(1)* %b) #0 {
 entry:
   %tid.x = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.y = call i32 @llvm.amdgcn.workitem.id.y()
@@ -38,3 +39,27 @@ entry:
   store i32 %tmp4, i32 addrspace(1)* %out
   ret void
 }
+
+; GCN-LABEL: {{^}}anyext_v2i16_to_v2i32:
+; GFX9: global_load_short_d16_hi
+; GFX9: v_and_b32_e32 v{{[0-9]+}}, 0x80008000
+; GFX9: v_bfi_b32 v{{[0-9]+}}, v{{[0-9]+}}, 0, v{{[0-9]+}}
+; GFX9: v_cmp_eq_f32_e32
+; GFX9: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
+define amdgpu_kernel void @anyext_v2i16_to_v2i32() #0 {
+bb:
+  %tmp = load i16, i16 addrspace(1)* undef, align 2
+  %tmp2 = insertelement <2 x i16> undef, i16 %tmp, i32 1
+  %tmp4 = and <2 x i16> %tmp2, <i16 -32768, i16 -32768>
+  %tmp5 = zext <2 x i16> %tmp4 to <2 x i32>
+  %tmp6 = shl nuw <2 x i32> %tmp5, <i32 16, i32 16>
+  %tmp7 = or <2 x i32> zeroinitializer, %tmp6
+  %tmp8 = bitcast <2 x i32> %tmp7 to <2 x float>
+  %tmp10 = fcmp oeq <2 x float> %tmp8, zeroinitializer
+  %tmp11 = zext <2 x i1> %tmp10 to <2 x i8>
+  %tmp12 = extractelement <2 x i8> %tmp11, i32 1
+  store i8 %tmp12, i8 addrspace(1)* undef, align 1
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll b/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
index daa3442097cf2..c1b8792c3d79a 100644
--- a/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
+++ b/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mattr=-promote-alloca < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mattr=+promote-alloca < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tahiti -mattr=+promote-alloca < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI %s
 
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #1
@@ -30,14 +30,14 @@ define amdgpu_kernel void @test_private_array_ptr_calc(i32 addrspace(1)* noalias
   %tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %mbcnt.lo)
   %a_ptr = getelementptr inbounds i32, i32 addrspace(1)* %inA, i32 %tid
   %b_ptr = getelementptr inbounds i32, i32 addrspace(1)* %inB, i32 %tid
-  %a = load i32, i32 addrspace(1)* %a_ptr
-  %b = load i32, i32 addrspace(1)* %b_ptr
+  %a = load i32, i32 addrspace(1)* %a_ptr, !range !0
+  %b = load i32, i32 addrspace(1)* %b_ptr, !range !0
   %result = add i32 %a, %b
   %alloca_ptr = getelementptr inbounds [16 x i32], [16 x i32]* %alloca, i32 1, i32 %b
   store i32 %result, i32* %alloca_ptr, align 4
   ; Dummy call
   call void @llvm.amdgcn.s.barrier()
-  %reload = load i32, i32* %alloca_ptr, align 4
+  %reload = load i32, i32* %alloca_ptr, align 4, !range !0
   %out_ptr = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid
   store i32 %reload, i32 addrspace(1)* %out_ptr, align 4
   ret void
@@ -46,3 +46,5 @@ define amdgpu_kernel void @test_private_array_ptr_calc(i32 addrspace(1)* noalias
 attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind convergent }
+
+!0 = !{i32 0, i32 65536 }
diff --git a/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll b/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
index ddeffc10a0895..3be0f3a74e5f3 100644
--- a/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
+++ b/test/CodeGen/AMDGPU/array-ptr-calc-i64.ll
@@ -4,8 +4,8 @@ declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #0
 
 ; SI-LABEL: {{^}}test_array_ptr_calc:
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_hi_i32
+; SI-DAG: v_mul_u32_u24
+; SI-DAG: v_mul_hi_u32_u24
 ; SI: s_endpgm
 define amdgpu_kernel void @test_array_ptr_calc(i32 addrspace(1)* noalias %out, [1025 x i32] addrspace(1)* noalias %inA, i32 addrspace(1)* noalias %inB) {
   %mbcnt.lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
diff --git a/test/CodeGen/AMDGPU/ashr.v2i16.ll b/test/CodeGen/AMDGPU/ashr.v2i16.ll
index dd96e6264418a..77bb582fee924 100644
--- a/test/CodeGen/AMDGPU/ashr.v2i16.ll
+++ b/test/CodeGen/AMDGPU/ashr.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=CIVI %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=CIVI %s
 
@@ -11,10 +11,10 @@
 ; VI: v_ashrrev_i32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
-; CI: v_ashrrev_i32_e32
-; CI: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
-; CI: v_ashrrev_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
+; CI-DAG: v_ashrrev_i32_e32
+; CI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
+; CI-DAG: v_ashrrev_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; CI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; CI: v_or_b32_e32
 define amdgpu_kernel void @s_ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
   %result = ashr <2 x i16> %lhs, %rhs
@@ -23,8 +23,8 @@ define amdgpu_kernel void @s_ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 }
 
 ; GCN-LABEL: {{^}}v_ashr_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 
 ; VI: v_ashrrev_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -56,7 +56,7 @@ define amdgpu_kernel void @v_ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 
 ; GCN-LABEL: {{^}}ashr_v_s_v2i16:
 ; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @ashr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -71,7 +71,7 @@ define amdgpu_kernel void @ashr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
 
 ; GCN-LABEL: {{^}}ashr_s_v_v2i16:
 ; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @ashr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -85,7 +85,7 @@ define amdgpu_kernel void @ashr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
 }
 
 ; GCN-LABEL: {{^}}ashr_imm_v_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], [[RHS]], -4
 define amdgpu_kernel void @ashr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -99,7 +99,7 @@ define amdgpu_kernel void @ashr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 }
 
 ; GCN-LABEL: {{^}}ashr_v_imm_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], 8, [[LHS]]
 define amdgpu_kernel void @ashr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -113,8 +113,8 @@ define amdgpu_kernel void @ashr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 }
 
 ; GCN-LABEL: {{^}}v_ashr_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
@@ -125,7 +125,7 @@ define amdgpu_kernel void @ashr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 ; VI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @v_ashr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -140,10 +140,10 @@ define amdgpu_kernel void @v_ashr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16>
 }
 
 ; GCN-LABEL: {{^}}ashr_v_imm_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @ashr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll b/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
index 4f9526ddab55f..836ba764a5b65 100644
--- a/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
+++ b/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
@@ -1,13 +1,17 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SICI -check-prefix=GCN -check-prefix=FUNC  %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=SICI -check-prefix=CIVI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=CIVI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI,SICI,SICIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SICI,CIVI,SICIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,CIVI,SICIVI,GFX89,GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX89,GCN %s
 
-; FUNC-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_offset:
-; SICI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; SICI: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xc
-; VI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; VI: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x30
-; GCN: v_mov_b32_e32 [[VCMP:v[0-9]+]], 7
+; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_offset:
+; GFX9-NOT: m0
+; SICIVI-DAG: s_mov_b32 m0
+
+; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
+; SICI-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xc
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; GFX89-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x30
+; GCN-DAG: v_mov_b32_e32 [[VCMP:v[0-9]+]], 7
 ; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
 ; GCN-DAG: v_mov_b32_e32 [[VSWAP:v[0-9]+]], [[SWAP]]
 ; GCN: ds_cmpst_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[VCMP]], [[VSWAP]] offset:16
@@ -20,18 +24,21 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_offset(i32 addrspace(1)* %
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_cmpxchg_ret_i64_offset:
+; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i64_offset:
+; GFX9-NOT: m0
+; SICIVI-DAG: s_mov_b32 m0
+
 ; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
 ; SICI-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xd
-; VI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; VI-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x34
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; GFX89-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x34
 ; GCN-DAG: v_mov_b32_e32 v[[LOVCMP:[0-9]+]], 7
 ; GCN-DAG: v_mov_b32_e32 v[[HIVCMP:[0-9]+]], 0
 ; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
 ; GCN-DAG: v_mov_b32_e32 v[[LOSWAPV:[0-9]+]], s[[LOSWAP]]
 ; GCN-DAG: v_mov_b32_e32 v[[HISWAPV:[0-9]+]], s[[HISWAP]]
 ; GCN: ds_cmpst_rtn_b64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v{{\[}}[[LOVCMP]]:[[HIVCMP]]{{\]}}, v{{\[}}[[LOSWAPV]]:[[HISWAPV]]{{\]}} offset:32
-; GCN: buffer_store_dwordx2 [[RESULT]],
+; GCN: [[RESULT]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr, i64 %swap) nounwind {
   %gep = getelementptr i64, i64 addrspace(3)* %ptr, i32 4
@@ -41,9 +48,11 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(i64 addrspace(1)* %
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_bad_si_offset
+; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_bad_si_offset
+; GFX9-NOT: m0
 ; SI: ds_cmpst_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CIVI: ds_cmpst_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
+; GFX9: ds_cmpst_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_bad_si_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr, i32 %swap, i32 %a, i32 %b) nounwind {
   %sub = sub i32 %a, %b
@@ -55,11 +64,15 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_bad_si_offset(i32 addrspac
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_cmpxchg_noret_i32_offset:
-; SICI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
-; SICI: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xa
-; VI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; VI: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x28
+; GCN-LABEL: {{^}}lds_atomic_cmpxchg_noret_i32_offset:
+; GFX9-NOT: m0
+; SICIVI-DAG: s_mov_b32 m0
+
+
+; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; SICI-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xa
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
+; GFX89-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x28
 ; GCN-DAG: v_mov_b32_e32 [[VCMP:v[0-9]+]], 7
 ; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
 ; GCN-DAG: v_mov_b32_e32 [[VSWAP:v[0-9]+]], [[SWAP]]
@@ -72,11 +85,14 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_noret_i32_offset(i32 addrspace(3)*
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_cmpxchg_noret_i64_offset:
-; SICI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
-; SICI: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; VI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; VI-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; GCN-LABEL: {{^}}lds_atomic_cmpxchg_noret_i64_offset:
+; GFX9-NOT: m0
+; SICIVI-DAG: s_mov_b32 m0
+
+; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; SICI-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xb
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
+; GFX89-DAG: s_load_dwordx2 s{{\[}}[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
 ; GCN-DAG: v_mov_b32_e32 v[[LOVCMP:[0-9]+]], 7
 ; GCN-DAG: v_mov_b32_e32 v[[HIVCMP:[0-9]+]], 0
 ; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
diff --git a/test/CodeGen/AMDGPU/atomic_load_add.ll b/test/CodeGen/AMDGPU/atomic_load_add.ll
index e0fe6641fa117..6b18fe0200d64 100644
--- a/test/CodeGen/AMDGPU/atomic_load_add.ll
+++ b/test/CodeGen/AMDGPU/atomic_load_add.ll
@@ -1,18 +1,24 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck %s -check-prefix=SI -check-prefix=FUNC
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=R600,FUNC %s
 
 ; FUNC-LABEL: {{^}}atomic_add_local:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; R600: LDS_ADD *
-; SI: ds_add_u32
+; GCN: ds_add_u32
 define amdgpu_kernel void @atomic_add_local(i32 addrspace(3)* %local) {
    %unused = atomicrmw volatile add i32 addrspace(3)* %local, i32 5 seq_cst
    ret void
 }
 
 ; FUNC-LABEL: {{^}}atomic_add_local_const_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_ADD *
-; SI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
+; GCN: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 define amdgpu_kernel void @atomic_add_local_const_offset(i32 addrspace(3)* %local) {
   %gep = getelementptr i32, i32 addrspace(3)* %local, i32 4
   %val = atomicrmw volatile add i32 addrspace(3)* %gep, i32 5 seq_cst
@@ -20,8 +26,11 @@ define amdgpu_kernel void @atomic_add_local_const_offset(i32 addrspace(3)* %loca
 }
 
 ; FUNC-LABEL: {{^}}atomic_add_ret_local:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_ADD_RET *
-; SI: ds_add_rtn_u32
+; GCN: ds_add_rtn_u32
 define amdgpu_kernel void @atomic_add_ret_local(i32 addrspace(1)* %out, i32 addrspace(3)* %local) {
   %val = atomicrmw volatile add i32 addrspace(3)* %local, i32 5 seq_cst
   store i32 %val, i32 addrspace(1)* %out
@@ -29,8 +38,11 @@ define amdgpu_kernel void @atomic_add_ret_local(i32 addrspace(1)* %out, i32 addr
 }
 
 ; FUNC-LABEL: {{^}}atomic_add_ret_local_const_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_ADD_RET *
-; SI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:20
+; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:20
 define amdgpu_kernel void @atomic_add_ret_local_const_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %local) {
   %gep = getelementptr i32, i32 addrspace(3)* %local, i32 5
   %val = atomicrmw volatile add i32 addrspace(3)* %gep, i32 5 seq_cst
diff --git a/test/CodeGen/AMDGPU/atomic_load_sub.ll b/test/CodeGen/AMDGPU/atomic_load_sub.ll
index a0275893919a9..1d6fe169e109a 100644
--- a/test/CodeGen/AMDGPU/atomic_load_sub.ll
+++ b/test/CodeGen/AMDGPU/atomic_load_sub.ll
@@ -1,18 +1,25 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -enable-var-scope -check-prefixes=R600,FUNC %s
 
 ; FUNC-LABEL: {{^}}atomic_sub_local:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_SUB *
-; SI: ds_sub_u32
+; GCN: ds_sub_u32
 define amdgpu_kernel void @atomic_sub_local(i32 addrspace(3)* %local) {
    %unused = atomicrmw volatile sub i32 addrspace(3)* %local, i32 5 seq_cst
    ret void
 }
 
 ; FUNC-LABEL: {{^}}atomic_sub_local_const_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_SUB *
-; SI: ds_sub_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
+; GCN: ds_sub_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 define amdgpu_kernel void @atomic_sub_local_const_offset(i32 addrspace(3)* %local) {
   %gep = getelementptr i32, i32 addrspace(3)* %local, i32 4
   %val = atomicrmw volatile sub i32 addrspace(3)* %gep, i32 5 seq_cst
@@ -20,8 +27,11 @@ define amdgpu_kernel void @atomic_sub_local_const_offset(i32 addrspace(3)* %loca
 }
 
 ; FUNC-LABEL: {{^}}atomic_sub_ret_local:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_SUB_RET *
-; SI: ds_sub_rtn_u32
+; GCN: ds_sub_rtn_u32
 define amdgpu_kernel void @atomic_sub_ret_local(i32 addrspace(1)* %out, i32 addrspace(3)* %local) {
   %val = atomicrmw volatile sub i32 addrspace(3)* %local, i32 5 seq_cst
   store i32 %val, i32 addrspace(1)* %out
@@ -29,8 +39,11 @@ define amdgpu_kernel void @atomic_sub_ret_local(i32 addrspace(1)* %out, i32 addr
 }
 
 ; FUNC-LABEL: {{^}}atomic_sub_ret_local_const_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; R600: LDS_SUB_RET *
-; SI: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:20
+; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:20
 define amdgpu_kernel void @atomic_sub_ret_local_const_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %local) {
   %gep = getelementptr i32, i32 addrspace(3)* %local, i32 5
   %val = atomicrmw volatile sub i32 addrspace(3)* %gep, i32 5 seq_cst
diff --git a/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
index a0694fb1e3c91..7fe5604c3ec72 100644
--- a/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
+++ b/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=HSAMD %s
 
 ; CHECK-LABEL: {{^}}min_64_max_64:
 ; CHECK: SGPRBlocks: 0
@@ -127,3 +128,15 @@ define amdgpu_kernel void @min_1024_max_2048() #3 {
   ret void
 }
 attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"}
+
+; HSAMD: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; HSAMD: Version: [ 1, 0 ]
+; HSAMD: Kernels:
+; HSAMD: - Name:                 min_64_max_64
+; HSAMD:   MaxFlatWorkGroupSize: 64
+; HSAMD: - Name:                 min_64_max_128
+; HSAMD:   MaxFlatWorkGroupSize: 128
+; HSAMD: - Name:                 min_128_max_128
+; HSAMD:   MaxFlatWorkGroupSize: 128
+; HSAMD: - Name:                 min_1024_max_2048
+; HSAMD:   MaxFlatWorkGroupSize: 2048
diff --git a/test/CodeGen/AMDGPU/basic-call-return.ll b/test/CodeGen/AMDGPU/basic-call-return.ll
new file mode 100644
index 0000000000000..1f2f92fbe5608
--- /dev/null
+++ b/test/CodeGen/AMDGPU/basic-call-return.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+define void @void_func_void() #2 {
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void:
+define amdgpu_kernel void @test_call_void_func_void() {
+  call void @void_func_void()
+  ret void
+}
+
+define void @void_func_void_clobber_s40_s41() #2 {
+  call void asm sideeffect "", "~{SGPR40_SGPR41}"() #0
+  ret void
+}
+
+define amdgpu_kernel void @test_call_void_func_void_clobber_s40_s41() {
+  call void @void_func_void_clobber_s40_s41()
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/basic-loop.ll b/test/CodeGen/AMDGPU/basic-loop.ll
index de45190cdaa56..d1dfd7c91dfb8 100644
--- a/test/CodeGen/AMDGPU/basic-loop.ll
+++ b/test/CodeGen/AMDGPU/basic-loop.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck %s
+; RUN: llc -O0 -verify-machineinstrs -march=amdgcn -mcpu=tahiti < %s | FileCheck %s
 ; RUN: llc -O0 -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}test_loop:
diff --git a/test/CodeGen/AMDGPU/bfe-combine.ll b/test/CodeGen/AMDGPU/bfe-combine.ll
index 6035e3bf4a5fe..cc0ae3ab4303e 100644
--- a/test/CodeGen/AMDGPU/bfe-combine.ll
+++ b/test/CodeGen/AMDGPU/bfe-combine.ll
@@ -9,8 +9,8 @@
 ; VI-SDWA: v_lshlrev_b32_sdwa v[[ADDRBASE:[0-9]+]], v[[SHIFT]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
 ; CI: v_lshrrev_b32_e32 v[[SHR:[0-9]+]], 6, v{{[0-9]+}}
 ; CI: v_and_b32_e32 v[[ADDRLO:[0-9]+]], 0x3fc, v[[SHR]]
-; VI: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
-; VI-SDWA: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
+; VI: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
+; VI-SDWA: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
 ; GCN: load_dword v{{[0-9]+}}, v{{\[}}[[ADDRLO]]:
 define amdgpu_kernel void @bfe_combine8(i32 addrspace(1)* nocapture %arg, i32 %x) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x() #2
@@ -29,11 +29,11 @@ define amdgpu_kernel void @bfe_combine8(i32 addrspace(1)* nocapture %arg, i32 %x
 ; VI-SDWA: v_mov_b32_e32 v[[SHIFT:[0-9]+]], 15
 ; VI-SDWA: v_lshlrev_b32_sdwa v[[ADDRBASE1:[0-9]+]], v[[SHIFT]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 ; VI-SDWA: v_lshlrev_b64 v{{\[}}[[ADDRBASE:[0-9]+]]:{{[^\]+}}], 2, v{{\[}}[[ADDRBASE1]]:{{[^\]+}}]
-; VI-SDWA: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
+; VI-SDWA: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
 ; CI: v_lshrrev_b32_e32 v[[SHR:[0-9]+]], 1, v{{[0-9]+}}
 ; CI: v_and_b32_e32 v[[AND:[0-9]+]], 0x7fff8000, v[[SHR]]
 ; CI: v_lshl_b64 v{{\[}}[[ADDRLO:[0-9]+]]:{{[^\]+}}], v{{\[}}[[AND]]:{{[^\]+}}], 2
-; VI: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
+; VI: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADDRBASE]]
 ; GCN: load_dword v{{[0-9]+}}, v{{\[}}[[ADDRLO]]:
 define amdgpu_kernel void @bfe_combine16(i32 addrspace(1)* nocapture %arg, i32 %x) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x() #2
diff --git a/test/CodeGen/AMDGPU/bfe-patterns.ll b/test/CodeGen/AMDGPU/bfe-patterns.ll
index 907c8c2216b76..a33ab4e379da5 100644
--- a/test/CodeGen/AMDGPU/bfe-patterns.ll
+++ b/test/CodeGen/AMDGPU/bfe-patterns.ll
@@ -22,7 +22,7 @@ define amdgpu_kernel void @v_ubfe_sub_i32(i32 addrspace(1)* %out, i32 addrspace(
 ; GCN-LABEL: {{^}}v_ubfe_sub_multi_use_shl_i32:
 ; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
-; GCN: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
+; GCN: v_sub_{{[iu]}}32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
 
 ; SI-NEXT: v_lshl_b32_e32 [[SHL:v[0-9]+]], [[SRC]], [[SUB]]
 ; SI-NEXT: v_lshr_b32_e32 [[BFE:v[0-9]+]], [[SHL]], [[SUB]]
@@ -100,7 +100,7 @@ define amdgpu_kernel void @v_sbfe_sub_i32(i32 addrspace(1)* %out, i32 addrspace(
 ; GCN-LABEL: {{^}}v_sbfe_sub_multi_use_shl_i32:
 ; GCN: {{buffer|flat}}_load_dword [[SRC:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[WIDTH:v[0-9]+]]
-; GCN: v_sub_i32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
+; GCN: v_sub_{{[iu]}}32_e32 [[SUB:v[0-9]+]], vcc, 32, [[WIDTH]]
 
 ; SI-NEXT: v_lshl_b32_e32 [[SHL:v[0-9]+]], [[SRC]], [[SUB]]
 ; SI-NEXT: v_ashr_i32_e32 [[BFE:v[0-9]+]], [[SHL]], [[SUB]]
diff --git a/test/CodeGen/AMDGPU/bitcast-vector-extract.ll b/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
index cf95f74afb84d..722099fec33c1 100644
--- a/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
+++ b/test/CodeGen/AMDGPU/bitcast-vector-extract.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; The bitcast should be pushed through the bitcasts so the vectors can
diff --git a/test/CodeGen/AMDGPU/br_cc.f16.ll b/test/CodeGen/AMDGPU/br_cc.f16.ll
index b7a0c8738dfaf..f145869f45862 100644
--- a/test/CodeGen/AMDGPU/br_cc.f16.ll
+++ b/test/CodeGen/AMDGPU/br_cc.f16.ll
@@ -1,23 +1,23 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}br_cc_f16:
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
 ; GCN: buffer_load_ushort v[[B_F16:[0-9]+]]
 
-; SI-DAG:  v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
-; SI-DAG:  v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
-; SI:  v_cmp_nlt_f32_e32 vcc, v[[B_F32]], v[[A_F32]]
+; SI:  v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
+; SI:  v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
+; SI:  v_cmp_nlt_f32_e32 vcc, v[[A_F32]], v[[B_F32]]
 ; VI:  v_cmp_nlt_f16_e32 vcc, v[[A_F16]], v[[B_F16]]
 ; GCN: s_cbranch_vccnz
 
 ; GCN: one{{$}}
-; SI: v_cvt_f16_f32_e32 v[[A_F16:[0-9]+]], v[[B_F32]]
+; SI: v_cvt_f16_f32_e32 v[[A_F16:[0-9]+]], v[[A_F32]]
 ; GCN: buffer_store_short
 ; GCN: s_endpgm
 
 ; GCN: two{{$}}
-; SI:  v_cvt_f16_f32_e32 v[[B_F16:[0-9]+]], v[[A_F32]]
+; SI:  v_cvt_f16_f32_e32 v[[B_F16:[0-9]+]], v[[B_F32]]
 ; GCN: buffer_store_short v[[B_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @br_cc_f16(
@@ -50,7 +50,7 @@ two:
 ; VI: s_cbranch_vccnz
 
 ; GCN: one{{$}}
-; VI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x380{{0|1}}{{$}}
+; GCN: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x380{{0|1}}{{$}}
 
 ; SI: buffer_store_short v[[A_F16]]
 ; SI: s_endpgm
@@ -89,7 +89,7 @@ two:
 ; SI:  v_cvt_f16_f32_e32 v[[A_F16:[0-9]+]], v[[A_F32]]
 
 ; GCN: two{{$}}
-; VI:  v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x3800{{$}}
+; GCN:  v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x3800{{$}}
 ; GCN: buffer_store_short v[[B_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @br_cc_f16_imm_b(
diff --git a/test/CodeGen/AMDGPU/branch-condition-and.ll b/test/CodeGen/AMDGPU/branch-condition-and.ll
index 662ea37a2b997..d1fcd1547d57d 100644
--- a/test/CodeGen/AMDGPU/branch-condition-and.ll
+++ b/test/CodeGen/AMDGPU/branch-condition-and.ll
@@ -14,14 +14,12 @@
 ; GCN-DAG: v_cmp_lt_f32_e32 vcc,
 ; GCN: s_and_b64 [[AND:s\[[0-9]+:[0-9]+\]]], vcc, [[OTHERCC]]
 ; GCN: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[AND]]
-; GCN: s_xor_b64 {{s\[[0-9]+:[0-9]+\]}}, exec, [[SAVED]]
 ; GCN: ; mask branch [[BB5:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT: BB{{[0-9]+_[0-9]+}}: ; %bb4
 ; GCN: ds_write_b32
 
 ; GCN: [[BB5]]
-; GCN: s_or_b64 exec, exec
 ; GCN-NEXT: s_endpgm
 ; GCN-NEXT: .Lfunc_end
 define amdgpu_ps void @ham(float %arg, float %arg1) #0 {
diff --git a/test/CodeGen/AMDGPU/branch-relax-bundle.ll b/test/CodeGen/AMDGPU/branch-relax-bundle.ll
new file mode 100644
index 0000000000000..9f0b8d3553ee0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/branch-relax-bundle.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-s-branch-bits=5 < %s | FileCheck -check-prefix=GCN %s
+
+; Restrict maximum branch to between +15 and -16 dwords
+
+; Instructions inside a bundle were collectively counted as
+; 0-bytes. Make sure this is accounted for when estimating branch
+; distances
+
+; Bundle used for address in call sequence: 20 bytes
+; s_getpc_b64
+; s_add_u32
+; s_addc_u32
+
+; plus additional overhead
+; s_setpc_b64
+; and some register copies
+
+declare void @func() #0
+
+; GCN-LABEL: {{^}}bundle_size:
+; GCN: s_cbranch_scc0 [[BB_EXPANSION:BB[0-9]+_[0-9]+]]
+; GCN: s_getpc_b64
+; GCN-NEXT: s_add_u32
+; GCN-NEXT: s_addc_u32
+; GCN-NEXT: s_setpc_b64
+
+; GCN: {{^}}[[BB_EXPANSION]]:
+; GCN: s_getpc_b64
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, func@
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, func@
+; GCN: s_swappc_b64
+define amdgpu_kernel void @bundle_size(i32 addrspace(1)* %arg, i32 %cnd) #0 {
+bb:
+  %cmp = icmp eq i32 %cnd, 0
+  br i1 %cmp, label %bb3, label %bb2 ; +8 dword branch
+
+bb2:
+  call void @func()
+  call void asm sideeffect
+  "v_nop_e64
+   v_nop_e64
+   v_nop_e64
+   v_nop_e64
+   v_nop_e64", ""() #0
+  br label %bb3
+
+bb3:
+  store volatile i32 %cnd, i32 addrspace(1)* %arg
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/branch-relaxation.ll b/test/CodeGen/AMDGPU/branch-relaxation.ll
index 15f579eb06d8f..023baf1407e77 100644
--- a/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs -amdgpu-s-branch-bits=4 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-s-branch-bits=4 < %s | FileCheck -check-prefix=GCN %s
 
 
 ; FIXME: We should use llvm-mc for this, but we can't even parse our own output.
@@ -24,7 +24,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
 ; GCN-NEXT: s_cbranch_scc1 [[BB3:BB[0-9]+_[0-9]+]]
 
 
-; GCN-NEXT: ; BB#1: ; %bb2
+; GCN-NEXT: ; %bb.1: ; %bb2
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: v_nop_e64
 ; GCN-NEXT: v_nop_e64
@@ -141,7 +141,6 @@ bb3:
 ; GCN: buffer_load_dword
 ; GCN: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
 ; GCN: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN: s_xor_b64 [[SAVE]], exec, [[SAVE]]
 
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
@@ -276,7 +275,7 @@ bb4:
 }
 
 ; GCN-LABEL: {{^}}uniform_unconditional_min_long_backward_branch:
-; GCN-NEXT: ; BB#0: ; %entry
+; GCN-NEXT: ; %bb.0: ; %entry
 
 ; GCN-NEXT: [[LOOP:BB[0-9]_[0-9]+]]: ; %loop
 ; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -312,7 +311,7 @@ loop:
 ; branch from %bb0 to %bb2
 
 ; GCN-LABEL: {{^}}expand_requires_expand:
-; GCN-NEXT: ; BB#0: ; %bb0
+; GCN-NEXT: ; %bb.0: ; %bb0
 ; GCN: s_load_dword
 ; GCN: s_cmp_lt_i32 s{{[0-9]+}}, 0{{$}}
 ; GCN-NEXT: s_cbranch_scc0 [[BB1:BB[0-9]+_[0-9]+]]
@@ -385,7 +384,6 @@ bb3:
 ; GCN-LABEL: {{^}}uniform_inside_divergent:
 ; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}}
 ; GCN-NEXT: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN-NEXT: s_xor_b64  [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]]
 ; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9]+_[0-9]+]]
 ; GCN-NEXT: s_cbranch_execnz [[IF:BB[0-9]+_[0-9]+]]
 
@@ -400,7 +398,7 @@ bb3:
 ; GCN: s_cmp_lg_u32
 ; GCN: s_cbranch_scc1 [[ENDIF]]
 
-; GCN-NEXT: ; BB#2: ; %if_uniform
+; GCN-NEXT: ; %bb.2: ; %if_uniform
 ; GCN: buffer_store_dword
 
 ; GCN-NEXT: [[ENDIF]]: ; %endif
@@ -430,25 +428,13 @@ endif:
 }
 
 ; si_mask_branch
-; s_cbranch_execz
-; s_branch
 
 ; GCN-LABEL: {{^}}analyze_mask_branch:
 ; GCN: v_cmp_lt_f32_e32 vcc
 ; GCN-NEXT: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN-NEXT: s_xor_b64 [[MASK]], exec, [[MASK]]
 ; GCN-NEXT: ; mask branch [[RET:BB[0-9]+_[0-9]+]]
-; GCN-NEXT: s_cbranch_execz [[BRANCH_SKIP:BB[0-9]+_[0-9]+]]
-; GCN-NEXT: s_branch [[LOOP_BODY:BB[0-9]+_[0-9]+]]
 
-; GCN-NEXT: [[BRANCH_SKIP]]: ; %entry
-; GCN-NEXT: s_getpc_b64 vcc
-; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[RET]]-([[BRANCH_SKIP]]+4)
-; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0
-; GCN-NEXT: s_setpc_b64 vcc
-
-; GCN-NEXT: [[LOOP_BODY]]: ; %loop_body
-; GCN: s_mov_b64 vcc, -1{{$}}
+; GCN-NEXT: [[LOOP_BODY:BB[0-9]+_[0-9]+]]: ; %loop_body
 ; GCN: ;;#ASMSTART
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
@@ -457,7 +443,6 @@ endif:
 ; GCN: v_nop_e64
 ; GCN: v_nop_e64
 ; GCN: ;;#ASMEND
-; GCN-NEXT: s_cbranch_vccz [[RET]]
 
 ; GCN-NEXT: [[LONGBB:BB[0-9]+_[0-9]+]]: ; %loop_body
 ; GCN-NEXT: ; in Loop: Header=[[LOOP_BODY]] Depth=1
@@ -466,7 +451,7 @@ endif:
 ; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0
 ; GCN-NEXT: s_setpc_b64 vcc
 
-; GCN-NEXT: [[RET]]: ; %Flow
+; GCN-NEXT: [[RET]]: ; %ret
 ; GCN-NEXT: s_or_b64 exec, exec, [[MASK]]
 ; GCN: buffer_store_dword
 ; GCN-NEXT: s_endpgm
diff --git a/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir b/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir
new file mode 100644
index 0000000000000..261a9e44d243c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/break-smem-soft-clauses.mir
@@ -0,0 +1,351 @@
+# RUN: llc -march=amdgcn -mcpu=carrizo -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,XNACK %s
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,NOXNACK %s
+
+---
+# Trivial clause at beginning of program
+name: trivial_smem_clause_load_smrd4_x1
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x1
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_smem_clause_load_smrd4_x2
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x2
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr1 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr1 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_smem_clause_load_smrd4_x3
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x3
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: %sgpr1 = S_LOAD_DWORD_IMM %sgpr6_sgpr7, 0, 0
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    %sgpr1 = S_LOAD_DWORD_IMM %sgpr6_sgpr7, 0, 0
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_smem_clause_load_smrd4_x4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x4
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: %sgpr1 = S_LOAD_DWORD_IMM %sgpr8_sgpr9, 0, 0
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    ; GCN-NEXT: %sgpr3 = S_LOAD_DWORD_IMM %sgpr16_sgpr17, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    %sgpr1 = S_LOAD_DWORD_IMM %sgpr8_sgpr9, 0, 0
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    %sgpr3 = S_LOAD_DWORD_IMM %sgpr16_sgpr17, 0, 0
+    S_ENDPGM
+...
+---
+# Reuse of same input pointer is OK
+name: trivial_smem_clause_load_smrd4_x2_sameptr
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_smem_clause_load_smrd4_x2_sameptr
+    ; GCN: %sgpr12 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr13 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr12 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr13 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    S_ENDPGM
+...
+---
+# 32-bit load partially clobbers its own ptr reg
+name: smrd_load4_overwrite_ptr_lo
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: smrd_load4_overwrite_ptr_lo
+    ; GCN: %sgpr10 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr10 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    S_ENDPGM
+...
+---
+# 32-bit load partially clobbers its own ptr reg
+name: smrd_load4_overwrite_ptr_hi
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: smrd_load4_overwrite_ptr_hi
+    ; GCN: %sgpr11 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr11 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    S_ENDPGM
+...
+---
+# 64-bit load clobbers its own ptr reg
+name: smrd_load8_overwrite_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: smrd_load8_overwrite_ptr
+    ; GCN: %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    S_ENDPGM
+...
+---
+# vmcnt has 4 bits, so maximum 16 outstanding loads. The waitcnt
+# breaks the clause.
+
+name: break_smem_clause_at_max_smem_clause_size_smrd_load4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_at_max_smem_clause_size_smrd_load4
+    ; GCN: %sgpr13 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr14 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr15 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr16 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr17 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr18 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr19 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr20 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr21 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr22 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr23 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr24 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr25 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr26 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr27 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr28 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr0 = S_LOAD_DWORD_IMM %sgpr30_sgpr31, 0, 0
+    ; GCN-NEXT: %sgpr0 = S_MOV_B32 %sgpr0, implicit %sgpr13, implicit %sgpr14, implicit %sgpr15, implicit %sgpr16, implicit %sgpr17, implicit %sgpr18, implicit %sgpr19, implicit %sgpr20, implicit %sgpr21, implicit %sgpr22, implicit %sgpr23, implicit %sgpr24, implicit %sgpr25, implicit %sgpr26, implicit %sgpr27, implicit %sgpr28
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr13 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr14 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr15 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr16 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+
+    %sgpr17 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr18 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr19 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr20 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+
+    %sgpr21 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr22 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr23 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr24 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+
+    %sgpr25 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr26 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr27 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr28 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr30_sgpr31, 0, 0
+    %sgpr0 = S_MOV_B32 %sgpr0, implicit %sgpr13, implicit %sgpr14, implicit %sgpr15, implicit %sgpr16, implicit %sgpr17, implicit %sgpr18, implicit %sgpr19, implicit %sgpr20, implicit %sgpr21, implicit %sgpr22, implicit %sgpr23, implicit %sgpr24, implicit %sgpr25, implicit %sgpr26, implicit %sgpr27, implicit %sgpr28
+    S_ENDPGM
+...
+---
+
+name: break_smem_clause_simple_load_smrd4_lo_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_simple_load_smrd4_lo_ptr
+    ; GCN: %sgpr10 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %sgpr12 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr10 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr12 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+
+name: break_smem_clause_simple_load_smrd4_hi_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_simple_load_smrd4_hi_ptr
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr3 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr3 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+
+name: break_smem_clause_simple_load_smrd8_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_simple_load_smrd8_ptr
+    ; GCN: %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+
+name: break_smem_clause_simple_load_smrd16_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_simple_load_smrd16_ptr
+    ; GCN: %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX4_IMM %sgpr6_sgpr7, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX4_IMM %sgpr6_sgpr7, 0, 0
+    S_ENDPGM
+...
+---
+
+name: break_smem_clause_block_boundary_load_smrd8_ptr
+
+body: |
+  ; GCN-LABEL: name: break_smem_clause_block_boundary_load_smrd8_ptr
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.1(0x80000000)
+  ; GCN:   %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+  ; GCN: bb.1:
+  ; XNACK-NEXT:   S_NOP 0
+  ; GCN-NEXT:   %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr12_sgpr13, 0, 0
+  ; GCN-NEXT:   S_ENDPGM
+  bb.0:
+    %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0
+
+  bb.1:
+    %sgpr10_sgpr11 = S_LOAD_DWORDX2_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+# The load clobbers the pointer of the store, so it needs to break.
+
+name: break_smem_clause_store_load_into_ptr_smrd4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_store_load_into_ptr_smrd4
+    ; GCN: S_STORE_DWORD_IMM %sgpr16, %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr12 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    S_STORE_DWORD_IMM %sgpr16, %sgpr10_sgpr11, 0, 0
+    %sgpr12 = S_LOAD_DWORD_IMM %sgpr14_sgpr15, 0, 0
+    S_ENDPGM
+...
+---
+# The load clobbers the data of the store, so it needs to break.
+# FIXME: Would it be better to s_nop and wait later?
+
+name: break_smem_clause_store_load_into_data_smrd4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_smem_clause_store_load_into_data_smrd4
+    ; GCN: S_STORE_DWORD_IMM %sgpr8, %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr8 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    S_STORE_DWORD_IMM %sgpr8, %sgpr10_sgpr11, 0, 0
+    %sgpr8 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+# Regular VALU instruction breaks clause, no nop needed
+name: valu_inst_breaks_smem_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: valu_inst_breaks_smem_clause
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %vgpr8 = V_MOV_B32_e32 0, implicit %exec
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %vgpr8 = V_MOV_B32_e32 0, implicit %exec
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+# Regular SALU instruction breaks clause, no nop needed
+name: salu_inst_breaks_smem_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: salu_inst_breaks_smem_clause
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %sgpr8 = S_MOV_B32 0
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %sgpr8 = S_MOV_B32 0
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+name: ds_inst_breaks_smem_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: ds_inst_breaks_smem_clause
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %vgpr8 = DS_READ_B32 %vgpr9, 0, 0, implicit %m0, implicit %exec
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %vgpr8 = DS_READ_B32 %vgpr9, 0, 0, implicit %m0, implicit %exec
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+
+name: flat_inst_breaks_smem_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: flat_inst_breaks_smem_clause
+    ; GCN: %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    ; GCN-NEXT: %vgpr0 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0 = S_LOAD_DWORD_IMM %sgpr10_sgpr11, 0, 0
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %sgpr2 = S_LOAD_DWORD_IMM %sgpr12_sgpr13, 0, 0
+    S_ENDPGM
+...
+---
+# FIXME: Should this be handled?
+name: implicit_use_breaks_smem_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: implicit_use_breaks_smem_clause
+    ; GCN: %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0, implicit %sgpr12_sgpr13
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %sgpr12_sgpr13 = S_LOAD_DWORDX2_IMM %sgpr6_sgpr7, 0, 0
+    ; GCN-NEXT: S_ENDPGM
+    %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM %sgpr10_sgpr11, 0, 0, implicit %sgpr12_sgpr13
+    %sgpr12_sgpr13 = S_LOAD_DWORDX2_IMM %sgpr6_sgpr7, 0, 0
+    S_ENDPGM
+...
diff --git a/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir b/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir
new file mode 100644
index 0000000000000..92145d319b193
--- /dev/null
+++ b/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir
@@ -0,0 +1,580 @@
+# RUN: llc -march=amdgcn -mcpu=carrizo -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,XNACK %s
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,NOXNACK %s
+---
+# Trivial clause at beginning of program
+name: trivial_clause_load_flat4_x1
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_flat4_x1
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_clause_load_flat4_x2
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_flat4_x2
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr1 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_clause_load_flat4_x3
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_flat4_x3
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr3_vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr1 = FLAT_LOAD_DWORD %vgpr5_vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr7_vgpr8, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr3_vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr5_vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr7_vgpr8, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Trivial clause at beginning of program
+name: trivial_clause_load_flat4_x4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_flat4_x4
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr1 = FLAT_LOAD_DWORD %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr8_vgpr9, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr3 = FLAT_LOAD_DWORD %vgpr10_vgpr11, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr8_vgpr9, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr3 = FLAT_LOAD_DWORD %vgpr10_vgpr11, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Reuse of same input pointer is OK
+
+name: trivial_clause_load_flat4_x2_sameptr
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_flat4_x2_sameptr
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr1 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# 32-bit load partially clobbers its own ptr reg
+name: flat_load4_overwrite_ptr_lo
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: flat_load4_overwrite_ptr_lo
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# 32-bit load partially clobbers its own ptr reg
+name: flat_load4_overwrite_ptr_hi
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: flat_load4_overwrite_ptr_hi
+    ; GCN: %vgpr1 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# 64-bit load clobbers its own ptr reg
+name: flat_load8_overwrite_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: flat_load8_overwrite_ptr
+    ; GCN: %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# vmcnt has 4 bits, so maximum 16 outstanding loads. The waitcnt
+# breaks the clause.
+
+
+name: break_clause_at_max_clause_size_flat_load4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_at_max_clause_size_flat_load4
+    ; GCN: %vgpr2 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr3 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr4 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr5 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr6 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr7 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr8 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr9 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr10 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr11 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr12 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr13 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr14 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr15 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr16 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr17 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %sgpr0 = S_MOV_B32 %sgpr0, implicit %vgpr2, implicit %vgpr3, implicit %vgpr4, implicit %vgpr5, implicit %vgpr6, implicit %vgpr7, implicit %vgpr8, implicit %vgpr9, implicit %vgpr10, implicit %vgpr11, implicit %vgpr12, implicit %vgpr13, implicit %vgpr14, implicit %vgpr15, implicit %vgpr16, implicit %vgpr17, implicit %vgpr18
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr3 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr4 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr5 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    %vgpr6 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr7 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr8 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr9 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    %vgpr10 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr11 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr12 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr13 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    %vgpr14 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr15 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr16 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr17 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %sgpr0 = S_MOV_B32 %sgpr0, implicit %vgpr2, implicit %vgpr3, implicit %vgpr4, implicit %vgpr5, implicit %vgpr6, implicit %vgpr7, implicit %vgpr8, implicit %vgpr9, implicit %vgpr10, implicit %vgpr11, implicit %vgpr12, implicit %vgpr13, implicit %vgpr14, implicit %vgpr15, implicit %vgpr16, implicit %vgpr17, implicit %vgpr18
+    S_ENDPGM
+...
+---
+
+name: break_clause_simple_load_flat4_lo_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_simple_load_flat4_lo_ptr
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+name: break_clause_simple_load_flat4_hi_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_simple_load_flat4_hi_ptr
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr3 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr3 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+name: break_clause_simple_load_flat8_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_simple_load_flat8_ptr
+    ; GCN: %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+
+name: break_clause_simple_load_flat16_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_simple_load_flat16_ptr
+    ; GCN: %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2_vgpr3_vgpr4_vgpr5 = FLAT_LOAD_DWORDX4 %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+    %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2_vgpr3_vgpr4_vgpr5 = FLAT_LOAD_DWORDX4 %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+# The clause is broken by the waitcnt inserted at the end of the
+# block, so no nop is needed.
+
+
+name: break_clause_block_boundary_load_flat8_ptr
+
+body: |
+  ; GCN-LABEL: name: break_clause_block_boundary_load_flat8_ptr
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.1(0x80000000)
+  ; GCN:   %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+  ; GCN: bb.1:
+  ; XNACK-NEXT:  S_NOP 0
+  ; GCN-NEXT:   %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+  ; GCN-NEXT:   S_ENDPGM
+
+  bb.0:
+    %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+  bb.1:
+    %vgpr2_vgpr3 = FLAT_LOAD_DWORDX2 %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# The load clobbers the pointer of the store, so it needs to break.
+
+name: break_clause_store_load_into_ptr_flat4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_store_load_into_ptr_flat4
+    ; GCN: FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# The load clobbers the data of the store, so it needs to break.
+# FIXME: Would it be better to s_nop and wait later?
+
+name: break_clause_store_load_into_data_flat4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_store_load_into_data_flat4
+    ; GCN: FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr0 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Regular VALU instruction breaks clause, no nop needed
+
+name: valu_inst_breaks_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: valu_inst_breaks_clause
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr8 = V_MOV_B32_e32 0, implicit %exec
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr8 = V_MOV_B32_e32 0, implicit %exec
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Regular SALU instruction breaks clause, no nop needed
+
+name: salu_inst_breaks_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: salu_inst_breaks_clause
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %sgpr8 = S_MOV_B32 0
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %sgpr8 = S_MOV_B32 0
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+name: ds_inst_breaks_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: ds_inst_breaks_clause
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr8 = DS_READ_B32 %vgpr9, 0, 0, implicit %m0, implicit %exec
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr8 = DS_READ_B32 %vgpr9, 0, 0, implicit %m0, implicit %exec
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+name: smrd_inst_breaks_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: smrd_inst_breaks_clause
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %sgpr8 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 0, 0
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %sgpr8 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 0, 0
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# FIXME: Should this be handled?
+name: implicit_use_breaks_clause
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: implicit_use_breaks_clause
+    ; GCN: %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr, implicit %vgpr4_vgpr5
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr4_vgpr5 = FLAT_LOAD_DWORDX2 %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr, implicit %vgpr4_vgpr5
+    %vgpr4_vgpr5 = FLAT_LOAD_DWORDX2 %vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+name: trivial_clause_load_mubuf4_x2
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: trivial_clause_load_mubuf4_x2
+    ; GCN: %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: %vgpr3 = BUFFER_LOAD_DWORD_OFFEN %vgpr4, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr3 = BUFFER_LOAD_DWORD_OFFEN %vgpr4, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    S_ENDPGM
+...
+---
+name: break_clause_simple_load_mubuf_offen_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_simple_load_mubuf_offen_ptr
+    ; GCN: %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = BUFFER_LOAD_DWORD_OFFEN %vgpr3, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr2 = BUFFER_LOAD_DWORD_OFFEN %vgpr3, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    S_ENDPGM
+...
+---
+# BUFFER instructions overwriting their own inputs is supposedly OK.
+
+name: mubuf_load4_overwrite_ptr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: mubuf_load4_overwrite_ptr
+    ; GCN: %vgpr0 = BUFFER_LOAD_DWORD_OFFEN %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: %vgpr1 = V_MOV_B32_e32 0, implicit %exec
+    ; GCN-NEXT: %vgpr2 = V_MOV_B32_e32 %vgpr0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+    %vgpr0 = BUFFER_LOAD_DWORD_OFFEN %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr1 = V_MOV_B32_e32 0, implicit %exec
+    %vgpr2 = V_MOV_B32_e32 %vgpr0, implicit %exec
+    S_ENDPGM
+...
+---
+# Break a clause from interference between mubuf and flat instructions
+
+name: break_clause_flat_load_mubuf_load
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_flat_load_mubuf_load
+    ; GCN: %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = BUFFER_LOAD_DWORD_OFFEN %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = BUFFER_LOAD_DWORD_OFFEN %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    S_ENDPGM
+...
+# Break a clause from interference between mubuf and flat instructions
+
+# GCN-LABEL: name: break_clause_mubuf_load_flat_load
+# GCN: bb.0:
+# GCN-NEXT: %vgpr0 = BUFFER_LOAD_DWORD_OFFEN %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4
+# XNACK-NEXT: S_NOP 0
+# GCN-NEXT: %vgpr1 = FLAT_LOAD_DWORD %vgpr2_vgpr3
+# GCN-NEXT: S_ENDPGM
+name: break_clause_mubuf_load_flat_load
+
+body: |
+  bb.0:
+    %vgpr0 = BUFFER_LOAD_DWORD_OFFEN %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr1 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    S_ENDPGM
+...
+---
+
+name: break_clause_atomic_rtn_into_ptr_flat4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_atomic_rtn_into_ptr_flat4
+    ; GCN: %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr4 = FLAT_ATOMIC_ADD_RTN %vgpr5_vgpr6, %vgpr7, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr4 = FLAT_ATOMIC_ADD_RTN %vgpr5_vgpr6, %vgpr7, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+name: break_clause_atomic_nortn_ptr_load_flat4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_atomic_nortn_ptr_load_flat4
+    ; GCN: FLAT_ATOMIC_ADD %vgpr0_vgpr1, %vgpr2, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr2 = FLAT_LOAD_DWORD %vgpr3_vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: S_ENDPGM
+
+    FLAT_ATOMIC_ADD %vgpr0_vgpr1, %vgpr2, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr2 = FLAT_LOAD_DWORD %vgpr3_vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+
+name: break_clause_atomic_rtn_into_ptr_mubuf4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_atomic_rtn_into_ptr_mubuf4
+    ; GCN: %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: %vgpr2 = BUFFER_ATOMIC_ADD_OFFEN_RTN %vgpr2, %vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+
+    %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr2 = BUFFER_ATOMIC_ADD_OFFEN_RTN %vgpr2, %vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, implicit %exec
+    S_ENDPGM
+...
+---
+
+name: break_clause_atomic_nortn_ptr_load_mubuf4
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: break_clause_atomic_nortn_ptr_load_mubuf4
+    ; GCN: BUFFER_ATOMIC_ADD_OFFEN %vgpr0, %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, implicit %exec
+    ; GCN-NEXT: %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+
+    BUFFER_ATOMIC_ADD_OFFEN %vgpr0, %vgpr1, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, implicit %exec
+    %vgpr1 = BUFFER_LOAD_DWORD_OFFEN %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    S_ENDPGM
+...
+---
+# Make sure there is no assert on mubuf instructions which do not have
+# vaddr, and don't add register to track.
+name: no_break_clause_mubuf_load_novaddr
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: no_break_clause_mubuf_load_novaddr
+    ; GCN: %vgpr1 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: %vgpr3 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    ; GCN-NEXT: S_ENDPGM
+    %vgpr1 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    %vgpr3 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec
+    S_ENDPGM
+...
+---
+# Loads and stores using different addresses theoretically does not
+# need a nop
+name: mix_load_store_clause
+body: |
+  bb.0:
+    ; GCN-LABEL: name: mix_load_store_clause
+    ; GCN: FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr10 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr11 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr10 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    FLAT_STORE_DWORD %vgpr2_vgpr3, %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr11 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
+---
+# Loads and stores using the same address needs a nop.
+
+name: mix_load_store_clause_same_address
+body: |
+  bb.0:
+    ; GCN-LABEL: name: mix_load_store_clause_same_address
+    ; GCN: FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr10 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; XNACK-NEXT: S_NOP 0
+    ; GCN-NEXT: FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    ; GCN-NEXT: %vgpr11 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+
+    FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr10 = FLAT_LOAD_DWORD %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr
+    FLAT_STORE_DWORD %vgpr0_vgpr1, %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %vgpr11 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr
+    S_ENDPGM
+...
diff --git a/test/CodeGen/AMDGPU/byval-frame-setup.ll b/test/CodeGen/AMDGPU/byval-frame-setup.ll
new file mode 100644
index 0000000000000..732142c12083f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/byval-frame-setup.ll
@@ -0,0 +1,212 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+
+%struct.ByValStruct = type { [4 x i32] }
+
+; GCN-LABEL: {{^}}void_func_byval_struct:
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_load_dword [[LOAD0:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-NOT: s32
+; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s5 offset:4{{$}}
+; GCN-NOT: s32
+
+; GCN: buffer_load_dword [[LOAD1:v[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN-NOT: s32
+; GCN: buffer_store_dword [[LOAD1]], off, s[0:3], s5 offset:20{{$}}
+; GCN-NOT: s32
+define void @void_func_byval_struct(%struct.ByValStruct* byval noalias nocapture align 4 %arg0, %struct.ByValStruct* byval noalias nocapture align 4 %arg1) #1 {
+entry:
+  %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg0, i32 0, i32 0, i32 0
+  %tmp = load volatile i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %tmp, 1
+  store volatile i32 %add, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg1, i32 0, i32 0, i32 0
+  %tmp1 = load volatile i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %tmp1, 2
+  store volatile i32 %add3, i32* %arrayidx2, align 4
+  store volatile i32 9, i32 addrspace(1)* null, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}void_func_byval_struct_non_leaf:
+; GCN: s_mov_b32 s5, s32
+; GCN-DAG: buffer_store_dword v32
+; GCN-DAG: buffer_store_dword v33
+; GCN-NOT: v_writelane_b32 v{{[0-9]+}}, s32
+; GCN: v_writelane_b32
+
+; GCN-DAG: s_add_u32 s32, s32, 0xb00{{$}}
+
+; GCN-DAG: buffer_load_dword [[LOAD0:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN: v_add_{{[iu]}}32_e32 [[ADD0:v[0-9]+]], vcc, 1, [[LOAD0]]
+; GCN: buffer_store_dword [[ADD0]], off, s[0:3], s5 offset:4{{$}}
+
+; GCN: buffer_load_dword [[LOAD1:v[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN: v_add_{{[iu]}}32_e32 [[ADD1:v[0-9]+]], vcc, 2, [[LOAD1]]
+
+; GCN: s_swappc_b64
+
+; GCN: buffer_store_dword [[ADD1]], off, s[0:3], s5 offset:20{{$}}
+
+; GCN: v_readlane_b32
+; GCN-NOT: v_readlane_b32 s32
+; GCN: buffer_load_dword v32,
+; GCN: buffer_load_dword v33,
+; GCN: s_sub_u32 s32, s32, 0xb00{{$}}
+; GCN: s_setpc_b64
+define void  @void_func_byval_struct_non_leaf(%struct.ByValStruct* byval noalias nocapture align 4 %arg0, %struct.ByValStruct* byval noalias nocapture align 4 %arg1) #1 {
+entry:
+  %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg0, i32 0, i32 0, i32 0
+  %tmp = load volatile i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %tmp, 1
+  store volatile i32 %add, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg1, i32 0, i32 0, i32 0
+  %tmp1 = load volatile i32, i32* %arrayidx2, align 4
+  %add3 = add nsw i32 %tmp1, 2
+  call void @external_void_func_void()
+  store volatile i32 %add3, i32* %arrayidx2, align 4
+  store volatile i32 9, i32 addrspace(1)* null, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}call_void_func_byval_struct_func:
+; GCN: s_mov_b32 s5, s32
+; GCN-DAG: s_add_u32 s32, s32, 0xc00{{$}}
+; GCN-DAG: v_writelane_b32
+
+; GCN-DAG: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
+; GCN-DAG: v_mov_b32_e32 [[THIRTEEN:v[0-9]+]], 13
+
+; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s5 offset:8
+; GCN-DAG: buffer_store_dword [[THIRTEEN]], off, s[0:3], s5 offset:24
+
+; GCN: buffer_load_dword [[LOAD0:v[0-9]+]], off, s[0:3], s5 offset:8
+; GCN: buffer_load_dword [[LOAD1:v[0-9]+]], off, s[0:3], s5 offset:12
+; GCN: buffer_load_dword [[LOAD2:v[0-9]+]], off, s[0:3], s5 offset:16
+; GCN: buffer_load_dword [[LOAD3:v[0-9]+]], off, s[0:3], s5 offset:20
+
+; GCN-NOT: s_add_u32 s32, s32, 0x800
+
+; GCN-DAG: buffer_store_dword [[LOAD0]], off, s[0:3], s32 offset:4{{$}}
+; GCN-DAG: buffer_store_dword [[LOAD1]], off, s[0:3], s32 offset:8
+; GCN-DAG: buffer_store_dword [[LOAD2]], off, s[0:3], s32 offset:12
+; GCN-DAG: buffer_store_dword [[LOAD3]], off, s[0:3], s32 offset:16
+
+; GCN: buffer_load_dword [[LOAD4:v[0-9]+]], off, s[0:3], s5 offset:24
+; GCN: buffer_load_dword [[LOAD5:v[0-9]+]], off, s[0:3], s5 offset:28
+; GCN: buffer_load_dword [[LOAD6:v[0-9]+]], off, s[0:3], s5 offset:32
+; GCN: buffer_load_dword [[LOAD7:v[0-9]+]], off, s[0:3], s5 offset:36
+
+; GCN-DAG: buffer_store_dword [[LOAD4]], off, s[0:3], s32 offset:20
+; GCN-DAG: buffer_store_dword [[LOAD5]], off, s[0:3], s32 offset:24
+; GCN-DAG: buffer_store_dword [[LOAD6]], off, s[0:3], s32 offset:28
+; GCN-DAG: buffer_store_dword [[LOAD7]], off, s[0:3], s32 offset:32
+
+; GCN: s_swappc_b64
+; GCN-NOT: v_readlane_b32 s32
+; GCN: v_readlane_b32
+; GCN-NOT: v_readlane_b32 s32
+
+; GCN-NOT: s_sub_u32 s32, s32, 0x800
+
+; GCN: s_sub_u32 s32, s32, 0xc00{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @call_void_func_byval_struct_func() #0 {
+entry:
+  %arg0 = alloca %struct.ByValStruct, align 4
+  %arg1 = alloca %struct.ByValStruct, align 4
+  %tmp = bitcast %struct.ByValStruct* %arg0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp)
+  %tmp1 = bitcast %struct.ByValStruct* %arg1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp1)
+  %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg0, i32 0, i32 0, i32 0
+  store volatile i32 9, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg1, i32 0, i32 0, i32 0
+  store volatile i32 13, i32* %arrayidx2, align 4
+  call void @void_func_byval_struct(%struct.ByValStruct* byval nonnull align 4 %arg0, %struct.ByValStruct* byval nonnull align 4 %arg1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp)
+  ret void
+}
+
+; GCN-LABEL: {{^}}call_void_func_byval_struct_kernel:
+; GCN: s_mov_b32 s33, s7
+; GCN: s_add_u32 s32, s33, 0xa00{{$}}
+
+; GCN-DAG: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
+; GCN-DAG: v_mov_b32_e32 [[THIRTEEN:v[0-9]+]], 13
+; GCN-DAG: buffer_store_dword [[NINE]], off, s[0:3], s33 offset:8
+; GCN: buffer_store_dword [[THIRTEEN]], off, s[0:3], s33 offset:24
+
+; GCN-NOT: s_add_u32 s32, s32, 0x800
+
+; GCN-DAG: buffer_load_dword [[LOAD0:v[0-9]+]], off, s[0:3], s33 offset:8
+; GCN-DAG: buffer_load_dword [[LOAD1:v[0-9]+]], off, s[0:3], s33 offset:12
+; GCN-DAG: buffer_load_dword [[LOAD2:v[0-9]+]], off, s[0:3], s33 offset:16
+; GCN-DAG: buffer_load_dword [[LOAD3:v[0-9]+]], off, s[0:3], s33 offset:20
+
+; GCN-DAG: buffer_store_dword [[LOAD0]], off, s[0:3], s32 offset:4{{$}}
+; GCN-DAG: buffer_store_dword [[LOAD1]], off, s[0:3], s32 offset:8
+; GCN-DAG: buffer_store_dword [[LOAD2]], off, s[0:3], s32 offset:12
+; GCN-DAG: buffer_store_dword [[LOAD3]], off, s[0:3], s32 offset:16
+
+; GCN-DAG: buffer_load_dword [[LOAD4:v[0-9]+]], off, s[0:3], s33 offset:24
+; GCN-DAG: buffer_load_dword [[LOAD5:v[0-9]+]], off, s[0:3], s33 offset:28
+; GCN-DAG: buffer_load_dword [[LOAD6:v[0-9]+]], off, s[0:3], s33 offset:32
+; GCN-DAG: buffer_load_dword [[LOAD7:v[0-9]+]], off, s[0:3], s33 offset:36
+
+; GCN-DAG: buffer_store_dword [[LOAD4]], off, s[0:3], s32 offset:20
+; GCN-DAG: buffer_store_dword [[LOAD5]], off, s[0:3], s32 offset:24
+; GCN-DAG: buffer_store_dword [[LOAD6]], off, s[0:3], s32 offset:28
+; GCN-DAG: buffer_store_dword [[LOAD7]], off, s[0:3], s32 offset:32
+
+
+; GCN: s_swappc_b64
+; GCN-NOT: s_sub_u32 s32
+; GCN: s_endpgm
+define amdgpu_kernel void @call_void_func_byval_struct_kernel() #0 {
+entry:
+  %arg0 = alloca %struct.ByValStruct, align 4
+  %arg1 = alloca %struct.ByValStruct, align 4
+  %tmp = bitcast %struct.ByValStruct* %arg0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp)
+  %tmp1 = bitcast %struct.ByValStruct* %arg1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp1)
+  %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg0, i32 0, i32 0, i32 0
+  store volatile i32 9, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg1, i32 0, i32 0, i32 0
+  store volatile i32 13, i32* %arrayidx2, align 4
+  call void @void_func_byval_struct(%struct.ByValStruct* byval nonnull align 4 %arg0, %struct.ByValStruct* byval nonnull align 4 %arg1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp)
+  ret void
+}
+
+; GCN-LABEL: {{^}}call_void_func_byval_struct_kernel_no_frame_pointer_elim:
+define amdgpu_kernel void @call_void_func_byval_struct_kernel_no_frame_pointer_elim() #2 {
+entry:
+  %arg0 = alloca %struct.ByValStruct, align 4
+  %arg1 = alloca %struct.ByValStruct, align 4
+  %tmp = bitcast %struct.ByValStruct* %arg0 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp)
+  %tmp1 = bitcast %struct.ByValStruct* %arg1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp1)
+  %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg0, i32 0, i32 0, i32 0
+  store volatile i32 9, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct* %arg1, i32 0, i32 0, i32 0
+  store volatile i32 13, i32* %arrayidx2, align 4
+  call void @void_func_byval_struct(%struct.ByValStruct* byval nonnull align 4 %arg0, %struct.ByValStruct* byval nonnull align 4 %arg1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp1)
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp)
+  ret void
+}
+
+declare void @external_void_func_void() #0
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #3
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #3
+
+attributes #0 = { nounwind }
+attributes #1 = { noinline norecurse nounwind }
+attributes #2 = { nounwind norecurse "no-frame-pointer-elim"="true" }
diff --git a/test/CodeGen/AMDGPU/call-argument-types.ll b/test/CodeGen/AMDGPU/call-argument-types.ll
new file mode 100644
index 0000000000000..e42546e60775b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -0,0 +1,527 @@
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,MESA %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,MESA %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,VI,MESA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,HSA %s
+
+declare void @external_void_func_i1(i1) #0
+declare void @external_void_func_i1_signext(i1 signext) #0
+declare void @external_void_func_i1_zeroext(i1 zeroext) #0
+
+declare void @external_void_func_i8(i8) #0
+declare void @external_void_func_i8_signext(i8 signext) #0
+declare void @external_void_func_i8_zeroext(i8 zeroext) #0
+
+declare void @external_void_func_i16(i16) #0
+declare void @external_void_func_i16_signext(i16 signext) #0
+declare void @external_void_func_i16_zeroext(i16 zeroext) #0
+
+declare void @external_void_func_i32(i32) #0
+declare void @external_void_func_i64(i64) #0
+
+declare void @external_void_func_f16(half) #0
+declare void @external_void_func_f32(float) #0
+declare void @external_void_func_f64(double) #0
+
+declare void @external_void_func_v2i16(<2 x i16>) #0
+declare void @external_void_func_v2f16(<2 x half>) #0
+
+declare void @external_void_func_v2i32(<2 x i32>) #0
+declare void @external_void_func_v3i32(<3 x i32>) #0
+declare void @external_void_func_v4i32(<4 x i32>) #0
+declare void @external_void_func_v8i32(<8 x i32>) #0
+declare void @external_void_func_v16i32(<16 x i32>) #0
+declare void @external_void_func_v32i32(<32 x i32>) #0
+declare void @external_void_func_v32i32_i32(<32 x i32>, i32) #0
+
+; return value and argument
+declare i32 @external_i32_func_i32(i32) #0
+
+; Structs
+declare void @external_void_func_struct_i8_i32({ i8, i32 }) #0
+declare void @external_void_func_byval_struct_i8_i32({ i8, i32 }* byval) #0
+declare void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 }* sret, { i8, i32 }* byval) #0
+
+declare void @external_void_func_v16i8(<16 x i8>) #0
+
+
+; FIXME: Should be passing -1
+; GCN-LABEL: {{^}}test_call_external_void_func_i1_imm:
+; MESA: s_mov_b32 s36, SCRATCH_RSRC_DWORD
+
+; MESA-DAG: s_mov_b64 s[0:1], s[36:37]
+
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i1@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i1@rel32@hi+4
+; GCN-DAG: v_mov_b32_e32 v0, 1{{$}}
+; MESA-DAG: s_mov_b64 s[2:3], s[38:39]
+
+; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i1_imm() #0 {
+  call void @external_void_func_i1(i1 true)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i1_signext:
+; MESA: s_mov_b32 s33, s3{{$}}
+; HSA: s_mov_b32 s33, s9{{$}}
+
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i1_signext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i1_signext@rel32@hi+4
+; GCN-NEXT: buffer_load_ubyte [[VAR:v[0-9]+]]
+; HSA-NEXT: s_mov_b32 s4, s33
+; HSA-NEXT: s_mov_b32 s32, s33
+
+; MESA-DAG: s_mov_b32 s4, s33{{$}}
+; MESA-DAG: s_mov_b32 s32, s33{{$}}
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i1_signext(i32) #0 {
+  %var = load volatile i1, i1 addrspace(1)* undef
+  call void @external_void_func_i1_signext(i1 %var)
+  ret void
+}
+
+; FIXME: load should be scheduled before getpc
+; GCN-LABEL: {{^}}test_call_external_void_func_i1_zeroext:
+; MESA: s_mov_b32 s33, s3{{$}}
+
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i1_zeroext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i1_zeroext@rel32@hi+4
+; GCN-NEXT: buffer_load_ubyte v0
+
+; GCN-DAG: s_mov_b32 s4, s33{{$}}
+; GCN-DAG: s_mov_b32 s32, s33{{$}}
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i1_zeroext(i32) #0 {
+  %var = load volatile i1, i1 addrspace(1)* undef
+  call void @external_void_func_i1_zeroext(i1 %var)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i8_imm:
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i8@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i8@rel32@hi+4
+; GCN-NEXT: v_mov_b32_e32 v0, 0x7b
+
+; HSA-DAG: s_mov_b32 s4, s33{{$}}
+; GCN-DAG: s_mov_b32 s32, s33{{$}}
+
+; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i8_imm(i32) #0 {
+  call void @external_void_func_i8(i8 123)
+  ret void
+}
+
+; FIXME: don't wait before call
+; GCN-LABEL: {{^}}test_call_external_void_func_i8_signext:
+; HSA-DAG: s_mov_b32 s33, s9{{$}}
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+; GCN-DAG: buffer_load_sbyte v0
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i8_signext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i8_signext@rel32@hi+4
+
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s3
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i8_signext(i32) #0 {
+  %var = load volatile i8, i8 addrspace(1)* undef
+  call void @external_void_func_i8_signext(i8 %var)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i8_zeroext:
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+; HSA-DAG: s_mov_b32 s33, s9{{$}}
+
+; GCN-DAG: buffer_load_ubyte v0
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i8_zeroext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i8_zeroext@rel32@hi+4
+
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s33
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i8_zeroext(i32) #0 {
+  %var = load volatile i8, i8 addrspace(1)* undef
+  call void @external_void_func_i8_zeroext(i8 %var)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i16_imm:
+; GCN-DAG: v_mov_b32_e32 v0, 0x7b{{$}}
+
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s33
+
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_i16_imm() #0 {
+  call void @external_void_func_i16(i16 123)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i16_signext:
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+; GCN-DAG: buffer_load_sshort v0
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i16_signext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i16_signext@rel32@hi+4
+
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s33
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i16_signext(i32) #0 {
+  %var = load volatile i16, i16 addrspace(1)* undef
+  call void @external_void_func_i16_signext(i16 %var)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i16_zeroext:
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+
+; GCN-DAG: buffer_load_ushort v0
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i16_zeroext@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i16_zeroext@rel32@hi+4
+
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s33
+
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i16_zeroext(i32) #0 {
+  %var = load volatile i16, i16 addrspace(1)* undef
+  call void @external_void_func_i16_zeroext(i16 %var)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i32_imm:
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-NEXT: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i32@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i32@rel32@hi+4
+; GCN: v_mov_b32_e32 v0, 42
+; GCN-DAG: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s32, s33
+
+; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i32_imm(i32) #0 {
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_i64_imm:
+; GCN-DAG: s_movk_i32 [[K0:s[0-9]+]], 0x7b{{$}}
+; GCN-DAG: s_mov_b32 [[K1:s[0-9]+]], 0{{$}}
+; GCN-DAG: v_mov_b32_e32 v0, [[K0]]
+; GCN-DAG: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; GCN-DAG: s_add_u32 s[[PC_LO]], s[[PC_LO]], external_void_func_i64@rel32@lo+4
+; GCN-DAG: s_addc_u32 s[[PC_HI]], s[[PC_HI]], external_void_func_i64@rel32@hi+4
+; GCN-DAG: v_mov_b32_e32 v1, [[K1]]
+; GCN: s_swappc_b64 s[30:31], s{{\[}}[[PC_LO]]:[[PC_HI]]{{\]}}
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_i64_imm() #0 {
+  call void @external_void_func_i64(i64 123)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_f16_imm:
+; VI: v_mov_b32_e32 v0, 0x4400
+; CI: v_mov_b32_e32 v0, 4.0
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_f16_imm() #0 {
+  call void @external_void_func_f16(half 4.0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_f32_imm:
+; GCN: v_mov_b32_e32 v0, 4.0
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_f32_imm() #0 {
+  call void @external_void_func_f32(float 4.0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_f64_imm:
+; GCN: v_mov_b32_e32 v0, 0{{$}}
+; GCN: v_mov_b32_e32 v1, 0x40100000
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_f64_imm() #0 {
+  call void @external_void_func_f64(double 4.0)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v2i16:
+; GFX9: buffer_load_dword v0
+; GFX9-NOT: v0
+; GFX9: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v2i16() #0 {
+  %val = load <2 x i16>, <2 x i16> addrspace(1)* undef
+  call void @external_void_func_v2i16(<2 x i16> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v2f16:
+; GFX9: buffer_load_dword v0
+; GFX9-NOT: v0
+; GFX9: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v2f16() #0 {
+  %val = load <2 x half>, <2 x half> addrspace(1)* undef
+  call void @external_void_func_v2f16(<2 x half> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v2i32:
+; GCN: buffer_load_dwordx2 v[0:1]
+; GCN: s_waitcnt
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v2i32() #0 {
+  %val = load <2 x i32>, <2 x i32> addrspace(1)* undef
+  call void @external_void_func_v2i32(<2 x i32> %val)
+  ret void
+}
+
+; FIXME: Passing 4th
+; GCN-LABEL: {{^}}test_call_external_void_func_v3i32_imm:
+; HSA-DAG: s_mov_b32 s33, s9
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+
+; GCN-DAG: v_mov_b32_e32 v0
+; GCN-DAG: v_mov_b32_e32 v1
+; GCN-DAG: v_mov_b32_e32 v2
+; GCN-DAG: v_mov_b32_e32 v3
+
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v3i32_imm(i32) #0 {
+  call void @external_void_func_v3i32(<3 x i32> <i32 3, i32 4, i32 5>)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v4i32:
+; GCN: buffer_load_dwordx4 v[0:3]
+; GCN: s_waitcnt
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v4i32() #0 {
+  %val = load <4 x i32>, <4 x i32> addrspace(1)* undef
+  call void @external_void_func_v4i32(<4 x i32> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v8i32:
+; GCN-DAG: buffer_load_dwordx4 v[0:3], off
+; GCN-DAG: buffer_load_dwordx4 v[4:7], off
+; GCN: s_waitcnt
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v8i32() #0 {
+  %ptr = load <8 x i32> addrspace(1)*, <8 x i32> addrspace(1)* addrspace(2)* undef
+  %val = load <8 x i32>, <8 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v8i32(<8 x i32> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v16i32:
+; GCN-DAG: buffer_load_dwordx4 v[0:3], off
+; GCN-DAG: buffer_load_dwordx4 v[4:7], off
+; GCN-DAG: buffer_load_dwordx4 v[8:11], off
+; GCN-DAG: buffer_load_dwordx4 v[12:15], off
+; GCN: s_waitcnt
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v16i32() #0 {
+  %ptr = load <16 x i32> addrspace(1)*, <16 x i32> addrspace(1)* addrspace(2)* undef
+  %val = load <16 x i32>, <16 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v16i32(<16 x i32> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v32i32:
+; GCN-DAG: buffer_load_dwordx4 v[0:3], off
+; GCN-DAG: buffer_load_dwordx4 v[4:7], off
+; GCN-DAG: buffer_load_dwordx4 v[8:11], off
+; GCN-DAG: buffer_load_dwordx4 v[12:15], off
+; GCN-DAG: buffer_load_dwordx4 v[16:19], off
+; GCN-DAG: buffer_load_dwordx4 v[20:23], off
+; GCN-DAG: buffer_load_dwordx4 v[24:27], off
+; GCN-DAG: buffer_load_dwordx4 v[28:31], off
+; GCN: s_waitcnt
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
+  %ptr = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(2)* undef
+  %val = load <32 x i32>, <32 x i32> addrspace(1)* %ptr
+  call void @external_void_func_v32i32(<32 x i32> %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v32i32_i32:
+; HSA-DAG: s_mov_b32 s33, s9
+; HSA-NOT: s_add_u32 s32
+
+; MESA-DAG: s_mov_b32 s33, s3{{$}}
+; MESA-NOT: s_add_u32 s32
+
+; GCN-DAG: buffer_load_dword [[VAL1:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-DAG: buffer_load_dwordx4 v[0:3], off
+; GCN-DAG: buffer_load_dwordx4 v[4:7], off
+; GCN-DAG: buffer_load_dwordx4 v[8:11], off
+; GCN-DAG: buffer_load_dwordx4 v[12:15], off
+; GCN-DAG: buffer_load_dwordx4 v[16:19], off
+; GCN-DAG: buffer_load_dwordx4 v[20:23], off
+; GCN-DAG: buffer_load_dwordx4 v[24:27], off
+; GCN-DAG: buffer_load_dwordx4 v[28:31], off
+
+; GCN: s_waitcnt
+; GCN: buffer_store_dword [[VAL1]], off, s[{{[0-9]+}}:{{[0-9]+}}], s32 offset:4{{$}}
+; GCN: s_swappc_b64
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
+  %ptr0 = load <32 x i32> addrspace(1)*, <32 x i32> addrspace(1)* addrspace(2)* undef
+  %val0 = load <32 x i32>, <32 x i32> addrspace(1)* %ptr0
+  %val1 = load i32, i32 addrspace(1)* undef
+  call void @external_void_func_v32i32_i32(<32 x i32> %val0, i32 %val1)
+  ret void
+}
+
+; FIXME: No wait after call
+; GCN-LABEL: {{^}}test_call_external_i32_func_i32_imm:
+; GCN: v_mov_b32_e32 v0, 42
+; GCN: s_swappc_b64 s[30:31],
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, off, s[36:39], 0
+define amdgpu_kernel void @test_call_external_i32_func_i32_imm(i32 addrspace(1)* %out) #0 {
+  %val = call i32 @external_i32_func_i32(i32 42)
+  store volatile i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_struct_i8_i32:
+; GCN: buffer_load_ubyte v0, off
+; GCN: buffer_load_dword v1, off
+; GCN: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_swappc_b64
+define amdgpu_kernel void @test_call_external_void_func_struct_i8_i32() #0 {
+  %ptr0 = load { i8, i32 } addrspace(1)*, { i8, i32 } addrspace(1)* addrspace(2)* undef
+  %val = load { i8, i32 }, { i8, i32 } addrspace(1)* %ptr0
+  call void @external_void_func_struct_i8_i32({ i8, i32 } %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_byval_struct_i8_i32:
+; GCN-DAG: s_add_u32 [[SP:s[0-9]+]], s33, 0x400{{$}}
+
+; GCN-DAG: v_mov_b32_e32 [[VAL0:v[0-9]+]], 3
+; GCN-DAG: v_mov_b32_e32 [[VAL1:v[0-9]+]], 8
+; MESA-DAG: buffer_store_byte [[VAL0]], off, s[36:39], s33 offset:8
+; MESA-DAG: buffer_store_dword [[VAL1]], off, s[36:39], s33 offset:12
+
+; HSA-DAG: buffer_store_byte [[VAL0]], off, s[0:3], s33 offset:8
+; HSA-DAG: buffer_store_dword [[VAL1]], off, s[0:3], s33 offset:12
+
+; GCN-NOT: s_add_u32 [[SP]],
+
+; HSA: buffer_load_dword [[RELOAD_VAL0:v[0-9]+]], off, s[0:3], s33 offset:8
+; HSA: buffer_load_dword [[RELOAD_VAL1:v[0-9]+]], off, s[0:3], s33 offset:12
+
+; HSA-DAG: buffer_store_dword [[RELOAD_VAL0]], off, s[0:3], [[SP]] offset:4
+; HSA-DAG: buffer_store_dword [[RELOAD_VAL1]], off, s[0:3], [[SP]] offset:8
+
+
+; MESA: buffer_load_dword [[RELOAD_VAL0:v[0-9]+]], off, s[36:39], s33 offset:8
+; MESA: buffer_load_dword [[RELOAD_VAL1:v[0-9]+]], off, s[36:39], s33 offset:12
+
+; MESA-DAG: buffer_store_dword [[RELOAD_VAL0]], off, s[36:39], [[SP]] offset:4
+; MESA-DAG: buffer_store_dword [[RELOAD_VAL1]], off, s[36:39], [[SP]] offset:8
+
+; GCN-NEXT: s_swappc_b64
+; GCN-NOT: [[SP]]
+define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0 {
+  %val = alloca { i8, i32 }, align 4
+  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %val, i32 0, i32 0
+  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %val, i32 0, i32 1
+  store i8 3, i8* %gep0
+  store i32 8, i32* %gep1
+  call void @external_void_func_byval_struct_i8_i32({ i8, i32 }* %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_sret_struct_i8_i32_byval_struct_i8_i32:
+; MESA-DAG: s_add_u32 [[SP:s[0-9]+]], [[FP_REG:s[0-9]+]], 0x600{{$}}
+; HSA-DAG: s_add_u32 [[SP:s[0-9]+]], [[FP_REG:s[0-9]+]], 0x600{{$}}
+
+; GCN-DAG: v_mov_b32_e32 [[VAL0:v[0-9]+]], 3
+; GCN-DAG: v_mov_b32_e32 [[VAL1:v[0-9]+]], 8
+; GCN-DAG: buffer_store_byte [[VAL0]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:8
+; GCN-DAG: buffer_store_dword [[VAL1]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:12
+
+; GCN-DAG: buffer_load_dword [[RELOAD_VAL0:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:8
+; GCN-DAG: buffer_load_dword [[RELOAD_VAL1:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:12
+
+; GCN-NOT: s_add_u32 [[SP]]
+; GCN-DAG: buffer_store_dword [[RELOAD_VAL0]], off, s{{\[[0-9]+:[0-9]+\]}}, [[SP]] offset:4
+; GCN-DAG: buffer_store_dword [[RELOAD_VAL1]], off, s{{\[[0-9]+:[0-9]+\]}}, [[SP]] offset:8
+; GCN-NEXT: s_swappc_b64
+; GCN-DAG: buffer_load_ubyte [[LOAD_OUT_VAL0:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:16
+; GCN-DAG: buffer_load_dword [[LOAD_OUT_VAL1:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, [[FP_REG]] offset:20
+; GCN-NOT: s_sub_u32 [[SP]]
+
+; GCN: buffer_store_byte [[LOAD_OUT_VAL0]], off
+; GCN: buffer_store_dword [[LOAD_OUT_VAL1]], off
+define amdgpu_kernel void @test_call_external_void_func_sret_struct_i8_i32_byval_struct_i8_i32(i32) #0 {
+  %in.val = alloca { i8, i32 }, align 4
+  %out.val = alloca { i8, i32 }, align 4
+  %in.gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %in.val, i32 0, i32 0
+  %in.gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %in.val, i32 0, i32 1
+  store i8 3, i8* %in.gep0
+  store i32 8, i32* %in.gep1
+  call void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 }* %out.val, { i8, i32 }* %in.val)
+  %out.gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %out.val, i32 0, i32 0
+  %out.gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %out.val, i32 0, i32 1
+  %out.val0 = load i8, i8* %out.gep0
+  %out.val1 = load i32, i32* %out.gep1
+
+  store volatile i8 %out.val0, i8 addrspace(1)* undef
+  store volatile i32 %out.val1, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_v16i8:
+define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
+  %ptr = load <16 x i8> addrspace(1)*, <16 x i8> addrspace(1)* addrspace(2)* undef
+  %val = load <16 x i8>, <16 x i8> addrspace(1)* %ptr
+  call void @external_void_func_v16i8(<16 x i8> %val)
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/call-encoding.ll b/test/CodeGen/AMDGPU/call-encoding.ll
new file mode 100644
index 0000000000000..f018e9b360df0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call-encoding.ll
@@ -0,0 +1,19 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -filetype=obj -verify-machineinstrs < %s | llvm-objdump -triple amdgcn--amdhsa -mcpu=fiji -d - | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -verify-machineinstrs < %s | llvm-objdump -triple amdgcn--amdhsa -mcpu=gfx900 -d - | FileCheck -check-prefixes=GCN,GFX9 %s
+; XUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -filetype=obj -verify-machineinstrs < %s | llvm-objdump -triple amdgcn--amdhsa -mcpu=hawaii -d - | FileCheck -check-prefixes=GCN,CI %s
+
+; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: s_setpc_b64
+define void @void_func_void() #1 {
+  ret void
+}
+
+; GCN: s_getpc_b64
+; GCN: s_swappc_b64
+define amdgpu_kernel void @test_call_void_func_void() {
+  call void @void_func_void()
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/call-graph-register-usage.ll b/test/CodeGen/AMDGPU/call-graph-register-usage.ll
new file mode 100644
index 0000000000000..58e549ef04a62
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call-graph-register-usage.ll
@@ -0,0 +1,230 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,VI-NOBUG %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=iceland -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,VI-BUG %s
+
+; Make sure to run a GPU with the SGPR allocation bug.
+
+; GCN-LABEL: {{^}}use_vcc:
+; GCN: ; NumSgprs: 34
+; GCN: ; NumVgprs: 0
+define void @use_vcc() #1 {
+  call void asm sideeffect "", "~{vcc}" () #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_vcc:
+; GCN: v_writelane_b32 v32, s33, 0
+; GCN: v_writelane_b32 v32, s34, 1
+; GCN: v_writelane_b32 v32, s35, 2
+; GCN: s_swappc_b64
+; GCN: v_readlane_b32 s35, v32, 2
+; GCN: v_readlane_b32 s34, v32, 1
+; GCN: v_readlane_b32 s33, v32, 0
+; GCN: ; NumSgprs: 38
+; GCN: ; NumVgprs: 33
+define void @indirect_use_vcc() #1 {
+  call void @use_vcc()
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_2level_use_vcc_kernel:
+; GCN: is_dynamic_callstack = 0
+; CI: ; NumSgprs: 40
+; VI-NOBUG: ; NumSgprs: 42
+; VI-BUG: ; NumSgprs: 96
+; GCN: ; NumVgprs: 33
+define amdgpu_kernel void @indirect_2level_use_vcc_kernel(i32 addrspace(1)* %out) #0 {
+  call void @indirect_use_vcc()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_flat_scratch:
+; CI: ; NumSgprs: 36
+; VI: ; NumSgprs: 38
+; GCN: ; NumVgprs: 0
+define void @use_flat_scratch() #1 {
+  call void asm sideeffect "", "~{flat_scratch}" () #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_flat_scratch:
+; CI: ; NumSgprs: 40
+; VI: ; NumSgprs: 42
+; GCN: ; NumVgprs: 33
+define void @indirect_use_flat_scratch() #1 {
+  call void @use_flat_scratch()
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_2level_use_flat_scratch_kernel:
+; GCN: is_dynamic_callstack = 0
+; CI: ; NumSgprs: 40
+; VI-NOBUG: ; NumSgprs: 42
+; VI-BUG: ; NumSgprs: 96
+; GCN: ; NumVgprs: 33
+define amdgpu_kernel void @indirect_2level_use_flat_scratch_kernel(i32 addrspace(1)* %out) #0 {
+  call void @indirect_use_flat_scratch()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_10_vgpr:
+; GCN: ; NumVgprs: 10
+define void @use_10_vgpr() #1 {
+  call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4}"() #0
+  call void asm sideeffect "", "~{v5},~{v6},~{v7},~{v8},~{v9}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_10_vgpr:
+; GCN: ; NumVgprs: 33
+define void @indirect_use_10_vgpr() #0 {
+  call void @use_10_vgpr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_2_level_use_10_vgpr:
+; GCN: is_dynamic_callstack = 0
+; GCN: ; NumVgprs: 10
+define amdgpu_kernel void @indirect_2_level_use_10_vgpr() #0 {
+  call void @indirect_use_10_vgpr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_40_vgpr:
+; GCN: ; NumVgprs: 40
+define void @use_40_vgpr() #1 {
+  call void asm sideeffect "", "~{v39}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_40_vgpr:
+; GCN: ; NumVgprs: 40
+define void @indirect_use_40_vgpr() #0 {
+  call void @use_40_vgpr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_80_sgpr:
+; GCN: ; NumSgprs: 80
+define void @use_80_sgpr() #1 {
+  call void asm sideeffect "", "~{s79}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_80_sgpr:
+; GCN: ; NumSgprs: 82
+define void @indirect_use_80_sgpr() #1 {
+  call void @use_80_sgpr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_2_level_use_80_sgpr:
+; GCN: is_dynamic_callstack = 0
+; CI: ; NumSgprs: 84
+; VI-NOBUG: ; NumSgprs: 86
+; VI-BUG: ; NumSgprs: 96
+define amdgpu_kernel void @indirect_2_level_use_80_sgpr() #0 {
+  call void @indirect_use_80_sgpr()
+  ret void
+}
+
+
+; GCN-LABEL: {{^}}use_stack0:
+; GCN: ScratchSize: 2052
+define void @use_stack0() #1 {
+  %alloca = alloca [512 x i32], align 4
+  call void asm sideeffect "; use $0", "v"([512 x i32]* %alloca) #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_stack1:
+; GCN: ScratchSize: 404
+define void @use_stack1() #1 {
+  %alloca = alloca [100 x i32], align 4
+  call void asm sideeffect "; use $0", "v"([100 x i32]* %alloca) #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_use_stack:
+; GCN: ScratchSize: 2124
+define void @indirect_use_stack() #1 {
+  %alloca = alloca [16 x i32], align 4
+  call void asm sideeffect "; use $0", "v"([16 x i32]* %alloca) #0
+  call void @use_stack0()
+  ret void
+}
+
+; GCN-LABEL: {{^}}indirect_2_level_use_stack:
+; GCN: is_dynamic_callstack = 0
+; GCN: ScratchSize: 2124
+define amdgpu_kernel void @indirect_2_level_use_stack() #0 {
+  call void @indirect_use_stack()
+  ret void
+}
+
+
+; Should be maximum of callee usage
+; GCN-LABEL: {{^}}multi_call_use_use_stack:
+; GCN: is_dynamic_callstack = 0
+; GCN: ScratchSize: 2052
+define amdgpu_kernel void @multi_call_use_use_stack() #0 {
+  call void @use_stack0()
+  call void @use_stack1()
+  ret void
+}
+
+
+declare void @external() #0
+
+; GCN-LABEL: {{^}}usage_external:
+; GCN: is_dynamic_callstack = 1
+; NumSgprs: 48
+; NumVgprs: 24
+; GCN: ScratchSize: 16384
+define amdgpu_kernel void @usage_external() #0 {
+  call void @external()
+  ret void
+}
+
+declare void @external_recurse() #2
+
+; GCN-LABEL: {{^}}usage_external_recurse:
+; GCN: is_dynamic_callstack = 1
+; NumSgprs: 48
+; NumVgprs: 24
+; GCN: ScratchSize: 16384
+define amdgpu_kernel void @usage_external_recurse() #0 {
+  call void @external_recurse()
+  ret void
+}
+
+; GCN-LABEL: {{^}}direct_recursion_use_stack:
+; GCN: ScratchSize: 2056
+define void @direct_recursion_use_stack(i32 %val) #2 {
+  %alloca = alloca [512 x i32], align 4
+  call void asm sideeffect "; use $0", "v"([512 x i32]* %alloca) #0
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %ret, label %call
+
+call:
+  %val.sub1 = sub i32 %val, 1
+  call void @direct_recursion_use_stack(i32 %val.sub1)
+  br label %ret
+
+ret:
+  ret void
+}
+
+; GCN-LABEL: {{^}}usage_direct_recursion:
+; GCN: is_ptr64 = 1
+; GCN: is_dynamic_callstack = 1
+; GCN: workitem_private_segment_byte_size = 2056
+define amdgpu_kernel void @usage_direct_recursion(i32 %n) #0 {
+  call void @direct_recursion_use_stack(i32 %n)
+  ret void
+}
+
+
+attributes #0 = { nounwind norecurse }
+attributes #1 = { nounwind noinline norecurse }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/call-preserved-registers.ll b/test/CodeGen/AMDGPU/call-preserved-registers.ll
new file mode 100644
index 0000000000000..093dd68d9f394
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call-preserved-registers.ll
@@ -0,0 +1,267 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare void @external_void_func_void() #0
+
+; GCN-LABEL: {{^}}test_kernel_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void:
+; GCN: s_mov_b32 s33, s7
+; GCN: s_getpc_b64 s[34:35]
+; GCN-NEXT: s_add_u32 s34, s34,
+; GCN-NEXT: s_addc_u32 s35, s35,
+; GCN-NEXT: s_mov_b32 s4, s33
+; GCN-NEXT: s_mov_b32 s32, s33
+; GCN: s_swappc_b64 s[30:31], s[34:35]
+
+; GCN-NEXT: s_mov_b32 s4, s33
+; GCN-NEXT: #ASMSTART
+; GCN-NEXT: #ASMEND
+; GCN-NEXT: s_swappc_b64 s[30:31], s[34:35]
+define amdgpu_kernel void @test_kernel_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
+  call void @external_void_func_void()
+  call void asm sideeffect "", ""() #0
+  call void @external_void_func_void()
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void:
+; GCN: v_writelane_b32 v32, s33, 0
+; GCN: v_writelane_b32 v32, s34, 1
+; GCN: v_writelane_b32 v32, s35, 2
+; GCN: v_writelane_b32 v32, s36, 3
+; GCN: v_writelane_b32 v32, s37, 4
+
+; GCN: s_mov_b32 s33, s5
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: s_mov_b32 s5, s33
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_mov_b32 s33, s5
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: s_mov_b32 s5, s33
+; GCN: v_readlane_b32 s37, v32, 4
+; GCN: v_readlane_b32 s36, v32, 3
+; GCN: v_readlane_b32 s35, v32, 2
+; GCN: v_readlane_b32 s34, v32, 1
+; GCN: v_readlane_b32 s33, v32, 0
+; GCN: s_setpc_b64
+define void @test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
+  call void @external_void_func_void()
+  call void asm sideeffect "", ""() #0
+  call void @external_void_func_void()
+  ret void
+}
+
+; FIXME: Avoid extra restore of FP in between calls.
+; GCN-LABEL: {{^}}test_func_call_external_void_funcx2:
+; GCN: s_mov_b32 s33, s5
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: s_mov_b32 s5, s33
+; GCN-NEXT: s_mov_b32 s33, s5
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: s_mov_b32 s5, s33
+define void @test_func_call_external_void_funcx2() #0 {
+  call void @external_void_func_void()
+  call void @external_void_func_void()
+  ret void
+}
+
+; GCN-LABEL: {{^}}void_func_void_clobber_s30_s31:
+; GCN: s_waitcnt
+; GCN-NEXT: s_mov_b64 [[SAVEPC:s\[[0-9]+:[0-9]+\]]], s[30:31]
+; GCN-NEXT: #ASMSTART
+; GCN: ; clobber
+; GCN-NEXT: #ASMEND
+; GCN-NEXT: s_mov_b64 s[30:31], [[SAVEPC]]
+; GCN-NEXT: s_setpc_b64 s[30:31]
+define void @void_func_void_clobber_s30_s31() #2 {
+  call void asm sideeffect "; clobber", "~{s[30:31]}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}void_func_void_clobber_vcc:
+; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_setpc_b64 s[30:31]
+define void @void_func_void_clobber_vcc() #2 {
+  call void asm sideeffect "", "~{VCC}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_clobber_vcc:
+; GCN: s_getpc_b64
+; GCN-NEXT: s_add_u32
+; GCN-NEXT: s_addc_u32
+; GCN: s_mov_b64 s[34:35], vcc
+; GCN-NEXT: s_swappc_b64
+; GCN: s_mov_b64 vcc, s[34:35]
+define amdgpu_kernel void @test_call_void_func_void_clobber_vcc(i32 addrspace(1)* %out) #0 {
+  %vcc = call i64 asm sideeffect "; def $0", "={vcc}"()
+  call void @void_func_void_clobber_vcc()
+  %val0 = load volatile i32, i32 addrspace(1)* undef
+  %val1 = load volatile i32, i32 addrspace(1)* undef
+  call void asm sideeffect "; use $0", "{vcc}"(i64 %vcc)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_mayclobber_s31:
+; GCN: s_mov_b32 s33, s31
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: s_mov_b32 s31, s33
+define amdgpu_kernel void @test_call_void_func_void_mayclobber_s31(i32 addrspace(1)* %out) #0 {
+  %s31 = call i32 asm sideeffect "; def $0", "={s31}"()
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "{s31}"(i32 %s31)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_mayclobber_v31:
+; GCN: v_mov_b32_e32 v32, v31
+; GCN-NEXT: s_swappc_b64
+; GCN-NEXT: v_mov_b32_e32 v31, v32
+define amdgpu_kernel void @test_call_void_func_void_mayclobber_v31(i32 addrspace(1)* %out) #0 {
+  %v31 = call i32 asm sideeffect "; def $0", "={v31}"()
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "{v31}"(i32 %v31)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_preserves_s33:
+; GCN: s_mov_b32 s34, s9
+; GCN: ; def s33
+; GCN-NEXT: #ASMEND
+; GCN: s_getpc_b64 s[6:7]
+; GCN-NEXT: s_add_u32 s6, s6, external_void_func_void@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s7, s7, external_void_func_void@rel32@hi+4
+; GCN-NEXT: s_mov_b32 s4, s34
+; GCN-NEXT: s_mov_b32 s32, s34
+; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7]
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use s33
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_void_func_void_preserves_s33(i32 addrspace(1)* %out) #0 {
+  %s33 = call i32 asm sideeffect "; def $0", "={s33}"()
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "{s33}"(i32 %s33)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_preserves_v32:
+; GCN: s_mov_b32 s33, s9
+; GCN: ; def v32
+; GCN-NEXT: #ASMEND
+; GCN: s_getpc_b64 s[6:7]
+; GCN-NEXT: s_add_u32 s6, s6, external_void_func_void@rel32@lo+4
+; GCN-NEXT: s_addc_u32 s7, s7, external_void_func_void@rel32@hi+4
+; GCN-NEXT: s_mov_b32 s4, s33
+; GCN-NEXT: s_mov_b32 s32, s33
+; GCN-NEXT: s_swappc_b64 s[30:31], s[6:7]
+; GCN-NEXT: ;;#ASMSTART
+; GCN-NEXT: ; use v32
+; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_void_func_void_preserves_v32(i32 addrspace(1)* %out) #0 {
+  %v32 = call i32 asm sideeffect "; def $0", "={v32}"()
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "{v32}"(i32 %v32)
+  ret void
+}
+
+; GCN-LABEL: {{^}}void_func_void_clobber_s33:
+; GCN: v_writelane_b32 v0, s33, 0
+; GCN-NEXT: #ASMSTART
+; GCN-NEXT: ; clobber
+; GCN-NEXT: #ASMEND
+; GCN-NEXT:	v_readlane_b32 s33, v0, 0
+; GCN-NEXT: s_setpc_b64
+define void @void_func_void_clobber_s33() #2 {
+  call void asm sideeffect "; clobber", "~{s33}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_void_func_void_clobber_s33:
+; GCN: s_mov_b32 s33, s7
+; GCN: s_getpc_b64
+; GCN-NEXT: s_add_u32
+; GCN-NEXT: s_addc_u32
+; GCN-NEXT: s_mov_b32 s4, s33
+; GCN-NEXT: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @test_call_void_func_void_clobber_s33() #0 {
+  call void @void_func_void_clobber_s33()
+  ret void
+}
+
+; GCN-LABEL: {{^}}callee_saved_sgpr_func:
+; GCN-NOT: s40
+; GCN: v_writelane_b32 v32, s40
+; GCN: s_swappc_b64
+; GCN-NOT: s40
+; GCN: ; use s40
+; GCN-NOT: s40
+; GCN: v_readlane_b32 s40, v32
+; GCN-NOT: s40
+define void @callee_saved_sgpr_func() #2 {
+  %s40 = call i32 asm sideeffect "; def s40", "={s40}"() #0
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "s"(i32 %s40) #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}callee_saved_sgpr_kernel:
+; GCN-NOT: s40
+; GCN: ; def s40
+; GCN-NOT: s40
+; GCN: s_swappc_b64
+; GCN-NOT: s40
+; GCN: ; use s40
+; GCN-NOT: s40
+define amdgpu_kernel void @callee_saved_sgpr_kernel() #2 {
+  %s40 = call i32 asm sideeffect "; def s40", "={s40}"() #0
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "s"(i32 %s40) #0
+  ret void
+}
+
+; First call preserved VGPR is used so it can't be used for SGPR spills.
+; GCN-LABEL: {{^}}callee_saved_sgpr_vgpr_func:
+; GCN-NOT: s40
+; GCN: v_writelane_b32 v33, s40
+; GCN: s_swappc_b64
+; GCN-NOT: s40
+; GCN: ; use s40
+; GCN-NOT: s40
+; GCN: v_readlane_b32 s40, v33
+; GCN-NOT: s40
+define void @callee_saved_sgpr_vgpr_func() #2 {
+  %s40 = call i32 asm sideeffect "; def s40", "={s40}"() #0
+  %v32 = call i32 asm sideeffect "; def v32", "={v32}"() #0
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "s"(i32 %s40) #0
+  call void asm sideeffect "; use $0", "v"(i32 %v32) #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}callee_saved_sgpr_vgpr_kernel:
+; GCN-NOT: s40
+; GCN: ; def s40
+; GCN-NOT: s40
+; GCN: s_swappc_b64
+; GCN-NOT: s40
+; GCN: ; use s40
+; GCN-NOT: s40
+define amdgpu_kernel void @callee_saved_sgpr_vgpr_kernel() #2 {
+  %s40 = call i32 asm sideeffect "; def s40", "={s40}"() #0
+  %v32 = call i32 asm sideeffect "; def v32", "={v32}"() #0
+  call void @external_void_func_void()
+  call void asm sideeffect "; use $0", "s"(i32 %s40) #0
+  call void asm sideeffect "; use $0", "v"(i32 %v32) #0
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/call-return-types.ll b/test/CodeGen/AMDGPU/call-return-types.ll
new file mode 100644
index 0000000000000..c6f893f15cbaf
--- /dev/null
+++ b/test/CodeGen/AMDGPU/call-return-types.ll
@@ -0,0 +1,241 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare void @external_void_func_void() #0
+
+declare i1 @external_i1_func_void() #0
+declare zeroext i1 @external_i1_zeroext_func_void() #0
+declare signext i1 @external_i1_signext_func_void() #0
+
+declare i8 @external_i8_func_void() #0
+declare zeroext i8 @external_i8_zeroext_func_void() #0
+declare signext i8 @external_i8_signext_func_void() #0
+
+declare i16 @external_i16_func_void() #0
+declare zeroext i16 @external_i16_zeroext_func_void() #0
+declare signext i16 @external_i16_signext_func_void() #0
+
+declare i32 @external_i32_func_void() #0
+declare i64 @external_i64_func_void() #0
+declare half @external_f16_func_void() #0
+declare float @external_f32_func_void() #0
+declare double @external_f64_func_void() #0
+
+declare <2 x i32> @external_v2i32_func_void() #0
+declare <3 x i32> @external_v3i32_func_void() #0
+declare <4 x i32> @external_v4i32_func_void() #0
+declare <5 x i32> @external_v5i32_func_void() #0
+declare <8 x i32> @external_v8i32_func_void() #0
+declare <16 x i32> @external_v16i32_func_void() #0
+declare <32 x i32> @external_v32i32_func_void() #0
+declare { <32 x i32>, i32 } @external_v32i32_i32_func_void() #0
+declare <2 x i16> @external_v2i16_func_void() #0
+declare <2 x half> @external_v2f16_func_void() #0
+
+declare { i32, i64 } @external_i32_i64_func_void() #0
+
+; GCN-LABEL: {{^}}test_call_external_void_func_void:
+define amdgpu_kernel void @test_call_external_void_func_void() #0 {
+  call void @external_void_func_void()
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_void_func_void_x2:
+define amdgpu_kernel void @test_call_external_void_func_void_x2() #0 {
+  call void @external_void_func_void()
+  call void @external_void_func_void()
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i1_func_void:
+define amdgpu_kernel void @test_call_external_i1_func_void() #0 {
+  %val = call i1 @external_i1_func_void()
+  store volatile i1 %val, i1 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i1_zeroext_func_void:
+define amdgpu_kernel void @test_call_external_i1_zeroext_func_void() #0 {
+  %val = call i1 @external_i1_zeroext_func_void()
+  %val.ext = zext i1 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i1_signext_func_void:
+define amdgpu_kernel void @test_call_external_i1_signext_func_void() #0 {
+  %val = call i1 @external_i1_signext_func_void()
+  %val.ext = zext i1 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i8_func_void:
+define amdgpu_kernel void @test_call_external_i8_func_void() #0 {
+  %val = call i8 @external_i8_func_void()
+  store volatile i8 %val, i8 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i8_zeroext_func_void:
+define amdgpu_kernel void @test_call_external_i8_zeroext_func_void() #0 {
+  %val = call i8 @external_i8_zeroext_func_void()
+  %val.ext = zext i8 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i8_signext_func_void:
+define amdgpu_kernel void @test_call_external_i8_signext_func_void() #0 {
+  %val = call i8 @external_i8_signext_func_void()
+  %val.ext = zext i8 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i16_func_void:
+define amdgpu_kernel void @test_call_external_i16_func_void() #0 {
+  %val = call i16 @external_i16_func_void()
+  store volatile i16 %val, i16 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i16_zeroext_func_void:
+define amdgpu_kernel void @test_call_external_i16_zeroext_func_void() #0 {
+  %val = call i16 @external_i16_zeroext_func_void()
+  %val.ext = zext i16 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i16_signext_func_void:
+define amdgpu_kernel void @test_call_external_i16_signext_func_void() #0 {
+  %val = call i16 @external_i16_signext_func_void()
+  %val.ext = zext i16 %val to i32
+  store volatile i32 %val.ext, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i32_func_void:
+define amdgpu_kernel void @test_call_external_i32_func_void() #0 {
+  %val = call i32 @external_i32_func_void()
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i64_func_void:
+define amdgpu_kernel void @test_call_external_i64_func_void() #0 {
+  %val = call i64 @external_i64_func_void()
+  store volatile i64 %val, i64 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_f16_func_void:
+define amdgpu_kernel void @test_call_external_f16_func_void() #0 {
+  %val = call half @external_f16_func_void()
+  store volatile half %val, half addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_f32_func_void:
+define amdgpu_kernel void @test_call_external_f32_func_void() #0 {
+  %val = call float @external_f32_func_void()
+  store volatile float %val, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_f64_func_void:
+define amdgpu_kernel void @test_call_external_f64_func_void() #0 {
+  %val = call double @external_f64_func_void()
+  store volatile double %val, double addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v2i32_func_void:
+define amdgpu_kernel void @test_call_external_v2i32_func_void() #0 {
+  %val = call <2 x i32> @external_v2i32_func_void()
+  store volatile <2 x i32> %val, <2 x i32> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v3i32_func_void:
+define amdgpu_kernel void @test_call_external_v3i32_func_void() #0 {
+  %val = call <3 x i32> @external_v3i32_func_void()
+  store volatile <3 x i32> %val, <3 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v4i32_func_void:
+define amdgpu_kernel void @test_call_external_v4i32_func_void() #0 {
+  %val = call <4 x i32> @external_v4i32_func_void()
+  store volatile <4 x i32> %val, <4 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v5i32_func_void:
+define amdgpu_kernel void @test_call_external_v5i32_func_void() #0 {
+  %val = call <5 x i32> @external_v5i32_func_void()
+  store volatile <5 x i32> %val, <5 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v8i32_func_void:
+define amdgpu_kernel void @test_call_external_v8i32_func_void() #0 {
+  %val = call <8 x i32> @external_v8i32_func_void()
+  store volatile <8 x i32> %val, <8 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v16i32_func_void:
+define amdgpu_kernel void @test_call_external_v16i32_func_void() #0 {
+  %val = call <16 x i32> @external_v16i32_func_void()
+  store volatile <16 x i32> %val, <16 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v32i32_func_void:
+define amdgpu_kernel void @test_call_external_v32i32_func_void() #0 {
+  %val = call <32 x i32> @external_v32i32_func_void()
+  store volatile <32 x i32> %val, <32 x i32> addrspace(1)* undef, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v2i16_func_void:
+define amdgpu_kernel void @test_call_external_v2i16_func_void() #0 {
+  %val = call <2 x i16> @external_v2i16_func_void()
+  store volatile <2 x i16> %val, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_v2f16_func_void:
+define amdgpu_kernel void @test_call_external_v2f16_func_void() #0 {
+  %val = call <2 x half> @external_v2f16_func_void()
+  store volatile <2 x half> %val, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_call_external_i32_i64_func_void:
+define amdgpu_kernel void @test_call_external_i32_i64_func_void() #0 {
+  %val = call { i32, i64 } @external_i32_i64_func_void()
+  %val.0 = extractvalue { i32, i64 } %val, 0
+  %val.1 = extractvalue { i32, i64 } %val, 1
+  store volatile i32 %val.0, i32 addrspace(1)* undef
+  store volatile i64 %val.1, i64 addrspace(1)* undef
+  ret void
+}
+
+; Requires writing results to stack
+; GCN-LABEL: {{^}}test_call_external_v32i32_i32_func_void:
+define amdgpu_kernel void @test_call_external_v32i32_i32_func_void() #0 {
+  %val = call { <32 x i32>, i32 } @external_v32i32_i32_func_void()
+  %val0 = extractvalue { <32 x i32>, i32 } %val, 0
+  %val1 = extractvalue { <32 x i32>, i32 } %val, 1
+  store volatile <32 x i32> %val0, <32 x i32> addrspace(1)* undef, align 8
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/call.ll b/test/CodeGen/AMDGPU/call.ll
deleted file mode 100644
index 769c7bb3eee7a..0000000000000
--- a/test/CodeGen/AMDGPU/call.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: not llc -march=amdgcn -verify-machineinstrs < %s 2>&1 | FileCheck %s
-; RUN: not llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s 2>&1 | FileCheck %s
-; RUN: not llc -march=r600 -mcpu=cypress < %s 2>&1 | FileCheck %s
-
-; CHECK: in function test_call_external{{.*}}: unsupported call to function external_function
-; CHECK: in function test_call{{.*}}: unsupported call to function defined_function
-; CHECK: in function test_tail_call{{.*}}: unsupported call to function defined_function
-; CHECK: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported call to function extern_variadic
-
-
-declare i32 @external_function(i32) nounwind
-
-define amdgpu_kernel void @test_call_external(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
-  %a = load i32, i32 addrspace(1)* %in
-  %b = load i32, i32 addrspace(1)* %b_ptr
-  %c = call i32 @external_function(i32 %b) nounwind
-  %result = add i32 %a, %c
-  store i32 %result, i32 addrspace(1)* %out
-  ret void
-}
-
-define i32 @defined_function(i32 %x) nounwind noinline {
-  %y = add i32 %x, 8
-  ret i32 %y
-}
-
-define amdgpu_kernel void @test_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
-  %a = load i32, i32 addrspace(1)* %in
-  %b = load i32, i32 addrspace(1)* %b_ptr
-  %c = call i32 @defined_function(i32 %b) nounwind
-  %result = add i32 %a, %c
-  store i32 %result, i32 addrspace(1)* %out
-  ret void
-}
-
-define amdgpu_kernel void @test_tail_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
-  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
-  %a = load i32, i32 addrspace(1)* %in
-  %b = load i32, i32 addrspace(1)* %b_ptr
-  %c = tail call i32 @defined_function(i32 %b) nounwind
-  %result = add i32 %a, %c
-  store i32 %result, i32 addrspace(1)* %out
-  ret void
-}
-
-define i32 @test_tail_call_ret() {
-  %call = call i32 @external_function(i32 10)
-  ret i32 %call
-}
-
-declare i32 @extern_variadic(...)
-
-define i32 @test_tail_call_bitcast_extern_variadic(<4 x float> %arg0, <4 x float> %arg1, i32 %arg2) {
-  %add = fadd <4 x float> %arg0, %arg1
-  %call = tail call i32 bitcast (i32 (...)* @extern_variadic to i32 (<4 x float>)*)(<4 x float> %add) #7
-  ret i32 %call
-}
diff --git a/test/CodeGen/AMDGPU/callee-frame-setup.ll b/test/CodeGen/AMDGPU/callee-frame-setup.ll
index 6c3594bb44ebf..88d165144f9d9 100644
--- a/test/CodeGen/AMDGPU/callee-frame-setup.ll
+++ b/test/CodeGen/AMDGPU/callee-frame-setup.ll
@@ -1,17 +1,26 @@
 ; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}callee_no_stack:
-; GCN: ; BB#0:
+; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @callee_no_stack() #0 {
   ret void
 }
 
+; GCN-LABEL: {{^}}callee_no_stack_no_fp_elim:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @callee_no_stack_no_fp_elim() #1 {
+  ret void
+}
+
 ; Requires frame pointer for access to local regular object.
 
 ; GCN-LABEL: {{^}}callee_with_stack:
-; GCN: ; BB#0:
+; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
 ; GCN-NEXT: s_mov_b32 s5, s32
 ; GCN-NEXT: v_mov_b32_e32 v0, 0{{$}}
@@ -24,4 +33,64 @@ define void @callee_with_stack() #0 {
   ret void
 }
 
+; GCN-LABEL: {{^}}callee_with_stack_and_call:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:8
+
+; GCN-DAG: v_writelane_b32 v32, s33,
+; GCN-DAG: v_writelane_b32 v32, s34,
+; GCN-DAG: v_writelane_b32 v32, s35,
+; GCN-DAG: s_add_u32 s32, s32, 0x300{{$}}
+; GCN-DAG: v_mov_b32_e32 v0, 0{{$}}
+; GCN: buffer_store_dword v0, off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: s_mov_b32 s33, s5
+
+
+; GCN: s_swappc_b64
+; GCN: s_mov_b32 s5, s33
+; GCN-DAG: v_readlane_b32 s35,
+; GCN-DAG: v_readlane_b32 s34,
+; GCN-DAG: v_readlane_b32 s33,
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8
+; GCN: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @callee_with_stack_and_call() #0 {
+  %alloca = alloca i32
+  store volatile i32 0, i32* %alloca
+  call void @external_void_func_void()
+  ret void
+}
+
+; Should be able to copy incoming stack pointer directly to inner
+; call's stack pointer argument.
+
+; There is stack usage only because of the need to evict a VGPR for
+; spilling CSR SGPRs.
+
+; GCN-LABEL: {{^}}callee_no_stack_with_call:
+; GCN: s_waitcnt
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:4
+; GCN-DAG: v_writelane_b32 v32, s33, 0
+; GCN-DAG: v_writelane_b32 v32, s34, 1
+; GCN: s_mov_b32 s33, s5
+; GCN: s_swappc_b64
+; GCN: s_mov_b32 s5, s33
+
+; GCN-DAG: v_readlane_b32 s34, v32, 1
+; GCN-DAG: v_readlane_b32 s33, v32, 0
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4
+; GCN: s_sub_u32 s32, s32, 0x200
+
+; GCN: s_setpc_b64
+define void @callee_no_stack_with_call() #0 {
+  call void @external_void_func_void()
+  ret void
+}
+
+declare void @external_void_func_void() #0
+
 attributes #0 = { nounwind }
+attributes #1 = { nounwind "no-frame-pointer-elim"="true" }
diff --git a/test/CodeGen/AMDGPU/callee-special-input-sgprs.ll b/test/CodeGen/AMDGPU/callee-special-input-sgprs.ll
new file mode 100644
index 0000000000000..8082a095a084e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/callee-special-input-sgprs.ll
@@ -0,0 +1,612 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}use_dispatch_ptr:
+; GCN: s_load_dword s{{[0-9]+}}, s[6:7], 0x0
+define void @use_dispatch_ptr() #1 {
+  %dispatch_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+  %header_ptr = bitcast i8 addrspace(2)* %dispatch_ptr to i32 addrspace(2)*
+  %value = load volatile i32, i32 addrspace(2)* %header_ptr
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_dispatch_ptr:
+; GCN: enable_sgpr_dispatch_ptr = 1
+; GCN: s_mov_b64 s[6:7], s[4:5]
+define amdgpu_kernel void @kern_indirect_use_dispatch_ptr(i32) #1 {
+  call void @use_dispatch_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_queue_ptr:
+; GCN: s_load_dword s{{[0-9]+}}, s[6:7], 0x0
+define void @use_queue_ptr() #1 {
+  %queue_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #0
+  %header_ptr = bitcast i8 addrspace(2)* %queue_ptr to i32 addrspace(2)*
+  %value = load volatile i32, i32 addrspace(2)* %header_ptr
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_queue_ptr:
+; GCN: enable_sgpr_queue_ptr = 1
+; GCN: s_mov_b64 s[6:7], s[4:5]
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_queue_ptr(i32) #1 {
+  call void @use_queue_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_queue_ptr_addrspacecast:
+; CIVI: s_load_dword [[APERTURE_LOAD:s[0-9]+]], s[6:7], 0x10
+; GFX9: s_getreg_b32 [[APERTURE_LOAD:s[0-9]+]]
+
+; GCN: v_mov_b32_e32 v[[HI:[0-9]+]], [[APERTURE_LOAD]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+}}:[[HI]]{{\]}}
+define void @use_queue_ptr_addrspacecast() #1 {
+  %asc = addrspacecast i32 addrspace(3)* inttoptr (i32 16 to i32 addrspace(3)*) to i32 addrspace(4)*
+  store volatile i32 0, i32 addrspace(4)* %asc
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_queue_ptr_addrspacecast:
+; CIVI: enable_sgpr_queue_ptr = 1
+
+; CIVI: s_mov_b64 s[6:7], s[4:5]
+; GFX9-NOT: s_mov_b64
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_queue_ptr_addrspacecast(i32) #1 {
+  call void @use_queue_ptr_addrspacecast()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_kernarg_segment_ptr:
+; GCN: s_load_dword s{{[0-9]+}}, s[6:7], 0x0
+define void @use_kernarg_segment_ptr() #1 {
+  %kernarg_segment_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0
+  %header_ptr = bitcast i8 addrspace(2)* %kernarg_segment_ptr to i32 addrspace(2)*
+  %value = load volatile i32, i32 addrspace(2)* %header_ptr
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_kernarg_segment_ptr:
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+; GCN: s_mov_b64 s[6:7], s[4:5]
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_kernarg_segment_ptr(i32) #1 {
+  call void @use_kernarg_segment_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_dispatch_id:
+; GCN: ; use s[6:7]
+define void @use_dispatch_id() #1 {
+  %id = call i64 @llvm.amdgcn.dispatch.id()
+  call void asm sideeffect "; use $0", "s"(i64 %id)
+  ret void
+}
+
+; No kernarg segment so that there is a mov to check. With kernarg
+; pointer enabled, it happens to end up in the right place anyway.
+
+; GCN-LABEL: {{^}}kern_indirect_use_dispatch_id:
+; GCN: enable_sgpr_dispatch_id = 1
+
+; GCN: s_mov_b64 s[6:7], s[4:5]
+define amdgpu_kernel void @kern_indirect_use_dispatch_id() #1 {
+  call void @use_dispatch_id()
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_x:
+; GCN: s_waitcnt
+; GCN: ; use s6
+define void @use_workgroup_id_x() #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.x()
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_stack_workgroup_id_x:
+; GCN: s_waitcnt
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v0, off, s[0:3], s5 offset:4
+; GCN: ; use s6
+; GCN: s_setpc_b64
+define void @use_stack_workgroup_id_x() #1 {
+  %alloca = alloca i32
+  store volatile i32 0, i32* %alloca
+  %val = call i32 @llvm.amdgcn.workgroup.id.x()
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_y:
+; GCN: s_waitcnt
+; GCN: ; use s6
+define void @use_workgroup_id_y() #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.y()
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_z:
+; GCN: s_waitcnt
+; GCN: ; use s6
+define void @use_workgroup_id_z() #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_xy:
+; GCN: ; use s6
+; GCN: ; use s7
+define void @use_workgroup_id_xy() #1 {
+  %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %val1 = call i32 @llvm.amdgcn.workgroup.id.y()
+  call void asm sideeffect "; use $0", "s"(i32 %val0)
+  call void asm sideeffect "; use $0", "s"(i32 %val1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_xyz:
+; GCN: ; use s6
+; GCN: ; use s7
+; GCN: ; use s8
+define void @use_workgroup_id_xyz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %val1 = call i32 @llvm.amdgcn.workgroup.id.y()
+  %val2 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val0)
+  call void asm sideeffect "; use $0", "s"(i32 %val1)
+  call void asm sideeffect "; use $0", "s"(i32 %val2)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_xz:
+; GCN: ; use s6
+; GCN: ; use s7
+define void @use_workgroup_id_xz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %val1 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val0)
+  call void asm sideeffect "; use $0", "s"(i32 %val1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workgroup_id_yz:
+; GCN: ; use s6
+; GCN: ; use s7
+define void @use_workgroup_id_yz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workgroup.id.y()
+  %val1 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val0)
+  call void asm sideeffect "; use $0", "s"(i32 %val1)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_x:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 0
+; GCN: enable_sgpr_workgroup_id_z = 0
+
+; GCN-NOT: s6
+; GCN: s_mov_b32 s33, s7
+; GCN-NOT: s6
+; GCN: s_mov_b32 s4, s33
+; GCN-NOT: s6
+; GCN: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_x() #1 {
+  call void @use_workgroup_id_x()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_y:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 0
+
+; GCN: s_mov_b32 s33, s8
+; GCN: s_mov_b32 s4, s33
+; GCN: s_mov_b32 s6, s7
+; GCN: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_y() #1 {
+  call void @use_workgroup_id_y()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_z:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 0
+; GCN: enable_sgpr_workgroup_id_z = 1
+
+; GCN: s_mov_b32 s33, s8
+; GCN: s_mov_b32 s4, s33
+; GCN: s_mov_b32 s6, s7
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_z() #1 {
+  call void @use_workgroup_id_z()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xy:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 0
+
+; GCN: s_mov_b32 s33, s8
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN: s_mov_b32 s4, s33
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN: s_mov_b32 s32, s33
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_xy() #1 {
+  call void @use_workgroup_id_xy()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xyz:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 1
+
+; GCN: s_mov_b32 s33, s9
+
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN-NOT: s8
+
+; GCN: s_mov_b32 s4, s33
+
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN-NOT: s8
+
+; GCN: s_mov_b32 s32, s33
+
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN-NOT: s8
+
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_xyz() #1 {
+  call void @use_workgroup_id_xyz()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_xz:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 0
+; GCN: enable_sgpr_workgroup_id_z = 1
+
+; GCN: s_mov_b32 s33, s8
+; GCN-NOT: s6
+; GCN-NOT: s7
+
+; GCN: s_mov_b32 s4, s33
+; GCN-NOT: s6
+; GCN-NOT: s7
+
+; GCN: s_mov_b32 s32, s33
+; GCN-NOT: s6
+; GCN-NOT: s7
+
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_xz() #1 {
+  call void @use_workgroup_id_xz()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workgroup_id_yz:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 1
+
+; GCN: s_mov_b32 s33, s9
+; GCN: s_mov_b32 s6, s7
+; GCN: s_mov_b32 s4, s33
+; GCN: s_mov_b32 s7, s8
+; GCN: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workgroup_id_yz() #1 {
+  call void @use_workgroup_id_yz()
+  ret void
+}
+
+; Argument is in right place already
+; GCN-LABEL: {{^}}func_indirect_use_workgroup_id_x:
+; GCN-NOT: s6
+define void @func_indirect_use_workgroup_id_x() #1 {
+  call void @use_workgroup_id_x()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_workgroup_id_y:
+; GCN-NOT: s6
+define void @func_indirect_use_workgroup_id_y() #1 {
+  call void @use_workgroup_id_y()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_workgroup_id_z:
+; GCN-NOT: s6
+define void @func_indirect_use_workgroup_id_z() #1 {
+  call void @use_workgroup_id_z()
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workgroup_id_x:
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN: ; use s6
+define void @other_arg_use_workgroup_id_x(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.x()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workgroup_id_y:
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN: ; use s6
+define void @other_arg_use_workgroup_id_y(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.y()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workgroup_id_z:
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN: ; use s6
+define void @other_arg_use_workgroup_id_z(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workgroup.id.z()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  call void asm sideeffect "; use $0", "s"(i32 %val)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workgroup_id_x:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 0
+; GCN: enable_sgpr_workgroup_id_z = 0
+
+; GCN-DAG: s_mov_b32 s33, s7
+; GCN-DAG: v_mov_b32_e32 v0, 0x22b
+
+; GCN-NOT: s6
+; GCN: s_mov_b32 s4, s33
+; GCN-NOT: s6
+; GCN-DAG: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_other_arg_use_workgroup_id_x() #1 {
+  call void @other_arg_use_workgroup_id_x(i32 555)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workgroup_id_y:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 0
+
+; GCN-DAG: s_mov_b32 s33, s8
+; GCN-DAG: v_mov_b32_e32 v0, 0x22b
+; GCN: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s6, s7
+; GCN-DAG: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_other_arg_use_workgroup_id_y() #1 {
+  call void @other_arg_use_workgroup_id_y(i32 555)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workgroup_id_z:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 0
+; GCN: enable_sgpr_workgroup_id_z = 1
+
+; GCN: s_mov_b32 s33, s8
+; GCN-DAG: v_mov_b32_e32 v0, 0x22b
+; GCN: s_mov_b32 s4, s33
+; GCN-DAG: s_mov_b32 s6, s7
+
+; GCN: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_other_arg_use_workgroup_id_z() #1 {
+  call void @other_arg_use_workgroup_id_z(i32 555)
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_every_sgpr_input:
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s5 offset:4
+; GCN: s_load_dword s{{[0-9]+}}, s[6:7], 0x0
+; GCN: s_load_dword s{{[0-9]+}}, s[8:9], 0x0
+; GCN: s_load_dword s{{[0-9]+}}, s[10:11], 0x0
+; GCN: ; use s[12:13]
+; GCN: ; use s14
+; GCN: ; use s15
+; GCN: ; use s16
+define void @use_every_sgpr_input() #1 {
+  %alloca = alloca i32, align 4
+  store volatile i32 0, i32* %alloca
+
+  %dispatch_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+  %dispatch_ptr.bc = bitcast i8 addrspace(2)* %dispatch_ptr to i32 addrspace(2)*
+  %val0 = load volatile i32, i32 addrspace(2)* %dispatch_ptr.bc
+
+  %queue_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #0
+  %queue_ptr.bc = bitcast i8 addrspace(2)* %queue_ptr to i32 addrspace(2)*
+  %val1 = load volatile i32, i32 addrspace(2)* %queue_ptr.bc
+
+  %kernarg_segment_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0
+  %kernarg_segment_ptr.bc = bitcast i8 addrspace(2)* %kernarg_segment_ptr to i32 addrspace(2)*
+  %val2 = load volatile i32, i32 addrspace(2)* %kernarg_segment_ptr.bc
+
+  %val3 = call i64 @llvm.amdgcn.dispatch.id()
+  call void asm sideeffect "; use $0", "s"(i64 %val3)
+
+  %val4 = call i32 @llvm.amdgcn.workgroup.id.x()
+  call void asm sideeffect "; use $0", "s"(i32 %val4)
+
+  %val5 = call i32 @llvm.amdgcn.workgroup.id.y()
+  call void asm sideeffect "; use $0", "s"(i32 %val5)
+
+  %val6 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val6)
+
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_every_sgpr_input:
+; GCN: enable_sgpr_workgroup_id_x = 1
+; GCN: enable_sgpr_workgroup_id_y = 1
+; GCN: enable_sgpr_workgroup_id_z = 1
+; GCN: enable_sgpr_workgroup_info = 0
+
+; GCN: enable_sgpr_private_segment_buffer = 1
+; GCN: enable_sgpr_dispatch_ptr = 1
+; GCN: enable_sgpr_queue_ptr = 1
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+; GCN: enable_sgpr_dispatch_id = 1
+; GCN: enable_sgpr_flat_scratch_init = 1
+
+; GCN: s_mov_b32 s33, s17
+; GCN: s_mov_b64 s[12:13], s[10:11]
+; GCN: s_mov_b64 s[10:11], s[8:9]
+; GCN: s_mov_b64 s[8:9], s[6:7]
+; GCN: s_mov_b64 s[6:7], s[4:5]
+; GCN: s_mov_b32 s4, s33
+; GCN: s_mov_b32 s32, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_every_sgpr_input() #1 {
+  call void @use_every_sgpr_input()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_every_sgpr_input:
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN-NOT: s8
+; GCN-NOT: s9
+; GCN-NOT: s10
+; GCN-NOT: s11
+; GCN-NOT: s12
+; GCN-NOT: s13
+; GCN-NOT: s[6:7]
+; GCN-NOT: s[8:9]
+; GCN-NOT: s[10:11]
+; GCN-NOT: s[12:13]
+define void @func_indirect_use_every_sgpr_input() #1 {
+  call void @use_every_sgpr_input()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_use_every_sgpr_input_call_use_workgroup_id_xyz:
+; GCN-DAG: s_mov_b32 s6, s14
+; GCN-DAG: s_mov_b32 s7, s15
+; GCN-DAG: s_mov_b32 s8, s16
+; GCN: s_swappc_b64
+define void @func_use_every_sgpr_input_call_use_workgroup_id_xyz() #1 {
+  %alloca = alloca i32, align 4
+  store volatile i32 0, i32* %alloca
+
+  %dispatch_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+  %dispatch_ptr.bc = bitcast i8 addrspace(2)* %dispatch_ptr to i32 addrspace(2)*
+  %val0 = load volatile i32, i32 addrspace(2)* %dispatch_ptr.bc
+
+  %queue_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #0
+  %queue_ptr.bc = bitcast i8 addrspace(2)* %queue_ptr to i32 addrspace(2)*
+  %val1 = load volatile i32, i32 addrspace(2)* %queue_ptr.bc
+
+  %kernarg_segment_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0
+  %kernarg_segment_ptr.bc = bitcast i8 addrspace(2)* %kernarg_segment_ptr to i32 addrspace(2)*
+  %val2 = load volatile i32, i32 addrspace(2)* %kernarg_segment_ptr.bc
+
+  %val3 = call i64 @llvm.amdgcn.dispatch.id()
+  call void asm sideeffect "; use $0", "s"(i64 %val3)
+
+  %val4 = call i32 @llvm.amdgcn.workgroup.id.x()
+  call void asm sideeffect "; use $0", "s"(i32 %val4)
+
+  %val5 = call i32 @llvm.amdgcn.workgroup.id.y()
+  call void asm sideeffect "; use $0", "s"(i32 %val5)
+
+  %val6 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val6)
+
+  call void @use_workgroup_id_xyz()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_use_every_sgpr_input_call_use_workgroup_id_xyz_spill:
+; GCN: s_mov_b32 s5, s32
+; GCN: s_add_u32 s32, s32, 0x300
+
+; GCN-DAG: s_mov_b32 [[SAVE_X:s[0-9]+]], s14
+; GCN-DAG: s_mov_b32 [[SAVE_Y:s[0-9]+]], s15
+; GCN-DAG: s_mov_b32 [[SAVE_Z:s[0-9]+]], s16
+; GCN-DAG: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, s[6:7]
+; GCN-DAG: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, s[8:9]
+; GCN-DAG: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, s[10:11]
+
+; GCN-DAG: s_mov_b32 s6, [[SAVE_X]]
+; GCN-DAG: s_mov_b32 s7, [[SAVE_Y]]
+; GCN-DAG: s_mov_b32 s8, [[SAVE_Z]]
+; GCN: s_swappc_b64
+
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s5 offset:4
+; GCN: s_load_dword s{{[0-9]+}},
+; GCN: s_load_dword s{{[0-9]+}},
+; GCN: s_load_dword s{{[0-9]+}},
+; GCN: ; use
+; GCN: ; use [[SAVE_X]]
+; GCN: ; use [[SAVE_Y]]
+; GCN: ; use [[SAVE_Z]]
+define void @func_use_every_sgpr_input_call_use_workgroup_id_xyz_spill() #1 {
+  %alloca = alloca i32, align 4
+  call void @use_workgroup_id_xyz()
+
+  store volatile i32 0, i32* %alloca
+
+  %dispatch_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+  %dispatch_ptr.bc = bitcast i8 addrspace(2)* %dispatch_ptr to i32 addrspace(2)*
+  %val0 = load volatile i32, i32 addrspace(2)* %dispatch_ptr.bc
+
+  %queue_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #0
+  %queue_ptr.bc = bitcast i8 addrspace(2)* %queue_ptr to i32 addrspace(2)*
+  %val1 = load volatile i32, i32 addrspace(2)* %queue_ptr.bc
+
+  %kernarg_segment_ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0
+  %kernarg_segment_ptr.bc = bitcast i8 addrspace(2)* %kernarg_segment_ptr to i32 addrspace(2)*
+  %val2 = load volatile i32, i32 addrspace(2)* %kernarg_segment_ptr.bc
+
+  %val3 = call i64 @llvm.amdgcn.dispatch.id()
+  call void asm sideeffect "; use $0", "s"(i64 %val3)
+
+  %val4 = call i32 @llvm.amdgcn.workgroup.id.x()
+  call void asm sideeffect "; use $0", "s"(i32 %val4)
+
+  %val5 = call i32 @llvm.amdgcn.workgroup.id.y()
+  call void asm sideeffect "; use $0", "s"(i32 %val5)
+
+  %val6 = call i32 @llvm.amdgcn.workgroup.id.z()
+  call void asm sideeffect "; use $0", "s"(i32 %val6)
+
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workgroup.id.x() #0
+declare i32 @llvm.amdgcn.workgroup.id.y() #0
+declare i32 @llvm.amdgcn.workgroup.id.z() #0
+declare noalias i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #0
+declare noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0
+declare i64 @llvm.amdgcn.dispatch.id() #0
+declare noalias i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll b/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
new file mode 100644
index 0000000000000..f5e68d188febb
--- /dev/null
+++ b/test/CodeGen/AMDGPU/callee-special-input-vgprs.ll
@@ -0,0 +1,671 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}use_workitem_id_x:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_x() #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_y:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_y() #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.y()
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_z:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_z() #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_xy:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_xy() #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.x()
+  %val1 = call i32 @llvm.amdgcn.workitem.id.y()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_xyz:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v2
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_xyz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.x()
+  %val1 = call i32 @llvm.amdgcn.workitem.id.y()
+  %val2 = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  store volatile i32 %val2, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_xz:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_xz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.x()
+  %val1 = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}use_workitem_id_yz:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]:[0-9]+\]}}, v1
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @use_workitem_id_yz() #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.y()
+  %val1 = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_x:
+; GCN: enable_vgpr_workitem_id = 0
+
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define amdgpu_kernel void @kern_indirect_use_workitem_id_x() #1 {
+  call void @use_workitem_id_x()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_y:
+; GCN: enable_vgpr_workitem_id = 1
+
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN: v_mov_b32_e32 v0, v1
+; GCN-NOT: v0
+; GCN-NOT: v1
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_y() #1 {
+  call void @use_workitem_id_y()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_use_workitem_id_z:
+; GCN: enable_vgpr_workitem_id = 2
+
+; GCN-NOT: v0
+; GCN-NOT: v2
+; GCN: v_mov_b32_e32 v0, v2
+; GCN-NOT: v0
+; GCN-NOT: v2
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_use_workitem_id_z() #1 {
+  call void @use_workitem_id_z()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_workitem_id_x:
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define void @func_indirect_use_workitem_id_x() #1 {
+  call void @use_workitem_id_x()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_workitem_id_y:
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define void @func_indirect_use_workitem_id_y() #1 {
+  call void @use_workitem_id_y()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_indirect_use_workitem_id_z:
+; GCN-NOT: v0
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define void @func_indirect_use_workitem_id_z() #1 {
+  call void @use_workitem_id_z()
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workitem_id_x:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+define void @other_arg_use_workitem_id_x(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workitem_id_y:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+define void @other_arg_use_workitem_id_y(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.y()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}other_arg_use_workitem_id_z:
+; GCN: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+define void @other_arg_use_workitem_id_z(i32 %arg0) #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %val, i32 addrspace(1)* undef
+  ret void
+}
+
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_x:
+; GCN: enable_vgpr_workitem_id = 0
+
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v0, 0x22b
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_x() #1 {
+  call void @other_arg_use_workitem_id_x(i32 555)
+  ret void
+}
+
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_y:
+; GCN: enable_vgpr_workitem_id = 1
+
+; GCN-NOT: v1
+; GCN: v_mov_b32_e32 v0, 0x22b
+; GCN-NOT: v1
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_y() #1 {
+  call void @other_arg_use_workitem_id_y(i32 555)
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_indirect_other_arg_use_workitem_id_z:
+; GCN: enable_vgpr_workitem_id = 2
+
+; GCN: v_mov_b32_e32 v0, 0x22b
+; GCN: v_mov_b32_e32 v1, v2
+; GCN: s_swappc_b64
+; GCN-NOT: v0
+define amdgpu_kernel void @kern_indirect_other_arg_use_workitem_id_z() #1 {
+  call void @other_arg_use_workitem_id_z(i32 555)
+  ret void
+}
+
+; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x:
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:8 ; 4-byte Folded Spill
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8 ; 4-byte Folded Reload
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @too_many_args_use_workitem_id_x(
+  i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+  i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+  i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+  i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31) #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, i32 addrspace(1)* undef
+
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %arg1, i32 addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(1)* undef
+  store volatile i32 %arg3, i32 addrspace(1)* undef
+  store volatile i32 %arg4, i32 addrspace(1)* undef
+  store volatile i32 %arg5, i32 addrspace(1)* undef
+  store volatile i32 %arg6, i32 addrspace(1)* undef
+  store volatile i32 %arg7, i32 addrspace(1)* undef
+
+  store volatile i32 %arg8, i32 addrspace(1)* undef
+  store volatile i32 %arg9, i32 addrspace(1)* undef
+  store volatile i32 %arg10, i32 addrspace(1)* undef
+  store volatile i32 %arg11, i32 addrspace(1)* undef
+  store volatile i32 %arg12, i32 addrspace(1)* undef
+  store volatile i32 %arg13, i32 addrspace(1)* undef
+  store volatile i32 %arg14, i32 addrspace(1)* undef
+  store volatile i32 %arg15, i32 addrspace(1)* undef
+
+  store volatile i32 %arg16, i32 addrspace(1)* undef
+  store volatile i32 %arg17, i32 addrspace(1)* undef
+  store volatile i32 %arg18, i32 addrspace(1)* undef
+  store volatile i32 %arg19, i32 addrspace(1)* undef
+  store volatile i32 %arg20, i32 addrspace(1)* undef
+  store volatile i32 %arg21, i32 addrspace(1)* undef
+  store volatile i32 %arg22, i32 addrspace(1)* undef
+  store volatile i32 %arg23, i32 addrspace(1)* undef
+
+  store volatile i32 %arg24, i32 addrspace(1)* undef
+  store volatile i32 %arg25, i32 addrspace(1)* undef
+  store volatile i32 %arg26, i32 addrspace(1)* undef
+  store volatile i32 %arg27, i32 addrspace(1)* undef
+  store volatile i32 %arg28, i32 addrspace(1)* undef
+  store volatile i32 %arg29, i32 addrspace(1)* undef
+  store volatile i32 %arg30, i32 addrspace(1)* undef
+  store volatile i32 %arg31, i32 addrspace(1)* undef
+
+  ret void
+}
+
+; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x:
+; GCN: enable_vgpr_workitem_id = 0
+
+; GCN: s_mov_b32 s33, s7
+; GCN: s_mov_b32 s32, s33
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:8
+; GCN: s_mov_b32 s4, s33
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x() #1 {
+  call void @too_many_args_use_workitem_id_x(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310, i32 320)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_too_many_args_use_workitem_id_x:
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v1, off, s[0:3], s32 offset:8
+; GCN: s_swappc_b64
+define void @func_call_too_many_args_use_workitem_id_x(i32 %arg0) #1 {
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  call void @too_many_args_use_workitem_id_x(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310, i32 320)
+  ret void
+}
+
+; Requires loading and storing to stack slot.
+; GCN-LABEL: {{^}}too_many_args_call_too_many_args_use_workitem_id_x:
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:8 ; 4-byte Folded Spill
+; GCN: s_add_u32 s32, s32, 0x400{{$}}
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4
+
+; GCN: buffer_store_dword v32, off, s[0:3], s32 offset:8{{$}}
+
+; GCN: s_swappc_b64
+
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8 ; 4-byte Folded Reload
+; GCN: s_sub_u32 s32, s32, 0x400{{$}}
+; GCN: s_setpc_b64
+define void @too_many_args_call_too_many_args_use_workitem_id_x(
+  i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+  i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+  i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+  i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31) #1 {
+  call void @too_many_args_use_workitem_id_x(
+    i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+    i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+    i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+    i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31)
+  ret void
+}
+
+; stack layout:
+; frame[0] = emergency stack slot
+; frame[1] = byval arg32
+; frame[2] = stack passed workitem ID x
+; frame[3] = VGPR spill slot
+
+; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x_byval:
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:12 ; 4-byte Folded Spill
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v32
+; GCN: buffer_load_dword v0, off, s[0:3], s5 offset:4
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:12 ; 4-byte Folded Reload
+; GCN: s_setpc_b64
+define void @too_many_args_use_workitem_id_x_byval(
+  i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+  i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+  i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+  i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31, i32* byval %arg32) #1 {
+  %val = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val, i32 addrspace(1)* undef
+
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %arg1, i32 addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(1)* undef
+  store volatile i32 %arg3, i32 addrspace(1)* undef
+  store volatile i32 %arg4, i32 addrspace(1)* undef
+  store volatile i32 %arg5, i32 addrspace(1)* undef
+  store volatile i32 %arg6, i32 addrspace(1)* undef
+  store volatile i32 %arg7, i32 addrspace(1)* undef
+
+  store volatile i32 %arg8, i32 addrspace(1)* undef
+  store volatile i32 %arg9, i32 addrspace(1)* undef
+  store volatile i32 %arg10, i32 addrspace(1)* undef
+  store volatile i32 %arg11, i32 addrspace(1)* undef
+  store volatile i32 %arg12, i32 addrspace(1)* undef
+  store volatile i32 %arg13, i32 addrspace(1)* undef
+  store volatile i32 %arg14, i32 addrspace(1)* undef
+  store volatile i32 %arg15, i32 addrspace(1)* undef
+
+  store volatile i32 %arg16, i32 addrspace(1)* undef
+  store volatile i32 %arg17, i32 addrspace(1)* undef
+  store volatile i32 %arg18, i32 addrspace(1)* undef
+  store volatile i32 %arg19, i32 addrspace(1)* undef
+  store volatile i32 %arg20, i32 addrspace(1)* undef
+  store volatile i32 %arg21, i32 addrspace(1)* undef
+  store volatile i32 %arg22, i32 addrspace(1)* undef
+  store volatile i32 %arg23, i32 addrspace(1)* undef
+
+  store volatile i32 %arg24, i32 addrspace(1)* undef
+  store volatile i32 %arg25, i32 addrspace(1)* undef
+  store volatile i32 %arg26, i32 addrspace(1)* undef
+  store volatile i32 %arg27, i32 addrspace(1)* undef
+  store volatile i32 %arg28, i32 addrspace(1)* undef
+  store volatile i32 %arg29, i32 addrspace(1)* undef
+  store volatile i32 %arg30, i32 addrspace(1)* undef
+  store volatile i32 %arg31, i32 addrspace(1)* undef
+  %private = load volatile i32, i32* %arg32
+  ret void
+}
+
+; frame[0] = emergency stack slot
+; frame[1] =
+
+; sp[0] = callee emergency stack slot reservation
+; sp[1] = byval
+; sp[2] = ??
+; sp[3] = stack passed workitem ID x
+
+; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_byval:
+; GCN: enable_vgpr_workitem_id = 0
+
+; GCN: s_mov_b32 s33, s7
+; GCN: s_add_u32 s32, s33, 0x200{{$}}
+
+; GCN-NOT: s32
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
+; GCN: buffer_store_dword [[K]], off, s[0:3], s33 offset:4
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:12
+
+; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s33 offset:4
+; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32 offset:4{{$}}
+; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x_byval() #1 {
+  %alloca = alloca i32, align 4
+  store volatile i32 999, i32* %alloca
+  call void @too_many_args_use_workitem_id_x_byval(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310, i32 320,
+    i32* %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_too_many_args_use_workitem_id_x_byval:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}}
+; GCN: buffer_store_dword [[K]], off, s[0:3], s5 offset:4
+; GCN: buffer_store_dword v0, off, s[0:3], s32 offset:12
+
+; GCN: buffer_load_dword [[RELOAD_BYVAL:v[0-9]+]], off, s[0:3], s5 offset:4
+; GCN: buffer_store_dword [[RELOAD_BYVAL]], off, s[0:3], s32 offset:4{{$}}
+; GCN: v_mov_b32_e32 [[RELOAD_BYVAL]],
+; GCN: s_swappc_b64
+define void @func_call_too_many_args_use_workitem_id_x_byval() #1 {
+  %alloca = alloca i32, align 4
+  store volatile i32 999, i32* %alloca
+  call void @too_many_args_use_workitem_id_x_byval(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310, i32 320,
+    i32* %alloca)
+  ret void
+}
+
+; GCN-LABEL: {{^}}too_many_args_use_workitem_id_xyz:
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:16 ; 4-byte Folded Spill
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:12{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v32
+
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:16 ; 4-byte Folded Reload
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @too_many_args_use_workitem_id_xyz(
+  i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+  i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+  i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+  i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30, i32 %arg31) #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  %val1 = call i32 @llvm.amdgcn.workitem.id.y()
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  %val2 = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val2, i32 addrspace(1)* undef
+
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %arg1, i32 addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(1)* undef
+  store volatile i32 %arg3, i32 addrspace(1)* undef
+  store volatile i32 %arg4, i32 addrspace(1)* undef
+  store volatile i32 %arg5, i32 addrspace(1)* undef
+  store volatile i32 %arg6, i32 addrspace(1)* undef
+  store volatile i32 %arg7, i32 addrspace(1)* undef
+
+  store volatile i32 %arg8, i32 addrspace(1)* undef
+  store volatile i32 %arg9, i32 addrspace(1)* undef
+  store volatile i32 %arg10, i32 addrspace(1)* undef
+  store volatile i32 %arg11, i32 addrspace(1)* undef
+  store volatile i32 %arg12, i32 addrspace(1)* undef
+  store volatile i32 %arg13, i32 addrspace(1)* undef
+  store volatile i32 %arg14, i32 addrspace(1)* undef
+  store volatile i32 %arg15, i32 addrspace(1)* undef
+
+  store volatile i32 %arg16, i32 addrspace(1)* undef
+  store volatile i32 %arg17, i32 addrspace(1)* undef
+  store volatile i32 %arg18, i32 addrspace(1)* undef
+  store volatile i32 %arg19, i32 addrspace(1)* undef
+  store volatile i32 %arg20, i32 addrspace(1)* undef
+  store volatile i32 %arg21, i32 addrspace(1)* undef
+  store volatile i32 %arg22, i32 addrspace(1)* undef
+  store volatile i32 %arg23, i32 addrspace(1)* undef
+
+  store volatile i32 %arg24, i32 addrspace(1)* undef
+  store volatile i32 %arg25, i32 addrspace(1)* undef
+  store volatile i32 %arg26, i32 addrspace(1)* undef
+  store volatile i32 %arg27, i32 addrspace(1)* undef
+  store volatile i32 %arg28, i32 addrspace(1)* undef
+  store volatile i32 %arg29, i32 addrspace(1)* undef
+  store volatile i32 %arg30, i32 addrspace(1)* undef
+  store volatile i32 %arg31, i32 addrspace(1)* undef
+
+  ret void
+}
+
+; frame[0] = kernel emergency stack slot
+; frame[1] = callee emergency stack slot
+; frame[2] = ID X
+; frame[3] = ID Y
+; frame[4] = ID Z
+
+; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_xyz:
+; GCN: enable_vgpr_workitem_id = 2
+
+; GCN: s_mov_b32 s33, s7
+; GCN: s_mov_b32 s32, s33
+
+; GCN-DAG: buffer_store_dword v0, off, s[0:3], s32 offset:8
+; GCN-DAG: buffer_store_dword v1, off, s[0:3], s32 offset:12
+; GCN-DAG: buffer_store_dword v2, off, s[0:3], s32 offset:16
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_xyz() #1 {
+  call void @too_many_args_use_workitem_id_xyz(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310, i32 320)
+  ret void
+}
+
+; workitem ID X in register, yz on stack
+; v31 = workitem ID X
+; frame[0] = emergency slot
+; frame[1] = workitem Y
+; frame[2] = workitem Z
+
+; GCN-LABEL: {{^}}too_many_args_use_workitem_id_x_stack_yz:
+; GCN: s_mov_b32 s5, s32
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
+; GCN: buffer_load_dword v31, off, s[0:3], s5 offset:4{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
+; GCN: buffer_load_dword v31, off, s[0:3], s5 offset:8{{$}}
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+]}}, v31
+
+; GCN: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+; GCN: ScratchSize: 12
+define void @too_many_args_use_workitem_id_x_stack_yz(
+  i32 %arg0, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6, i32 %arg7,
+  i32 %arg8, i32 %arg9, i32 %arg10, i32 %arg11, i32 %arg12, i32 %arg13, i32 %arg14, i32 %arg15,
+  i32 %arg16, i32 %arg17, i32 %arg18, i32 %arg19, i32 %arg20, i32 %arg21, i32 %arg22, i32 %arg23,
+  i32 %arg24, i32 %arg25, i32 %arg26, i32 %arg27, i32 %arg28, i32 %arg29, i32 %arg30) #1 {
+  %val0 = call i32 @llvm.amdgcn.workitem.id.x()
+  store volatile i32 %val0, i32 addrspace(1)* undef
+  %val1 = call i32 @llvm.amdgcn.workitem.id.y()
+  store volatile i32 %val1, i32 addrspace(1)* undef
+  %val2 = call i32 @llvm.amdgcn.workitem.id.z()
+  store volatile i32 %val2, i32 addrspace(1)* undef
+
+  store volatile i32 %arg0, i32 addrspace(1)* undef
+  store volatile i32 %arg1, i32 addrspace(1)* undef
+  store volatile i32 %arg2, i32 addrspace(1)* undef
+  store volatile i32 %arg3, i32 addrspace(1)* undef
+  store volatile i32 %arg4, i32 addrspace(1)* undef
+  store volatile i32 %arg5, i32 addrspace(1)* undef
+  store volatile i32 %arg6, i32 addrspace(1)* undef
+  store volatile i32 %arg7, i32 addrspace(1)* undef
+
+  store volatile i32 %arg8, i32 addrspace(1)* undef
+  store volatile i32 %arg9, i32 addrspace(1)* undef
+  store volatile i32 %arg10, i32 addrspace(1)* undef
+  store volatile i32 %arg11, i32 addrspace(1)* undef
+  store volatile i32 %arg12, i32 addrspace(1)* undef
+  store volatile i32 %arg13, i32 addrspace(1)* undef
+  store volatile i32 %arg14, i32 addrspace(1)* undef
+  store volatile i32 %arg15, i32 addrspace(1)* undef
+
+  store volatile i32 %arg16, i32 addrspace(1)* undef
+  store volatile i32 %arg17, i32 addrspace(1)* undef
+  store volatile i32 %arg18, i32 addrspace(1)* undef
+  store volatile i32 %arg19, i32 addrspace(1)* undef
+  store volatile i32 %arg20, i32 addrspace(1)* undef
+  store volatile i32 %arg21, i32 addrspace(1)* undef
+  store volatile i32 %arg22, i32 addrspace(1)* undef
+  store volatile i32 %arg23, i32 addrspace(1)* undef
+
+  store volatile i32 %arg24, i32 addrspace(1)* undef
+  store volatile i32 %arg25, i32 addrspace(1)* undef
+  store volatile i32 %arg26, i32 addrspace(1)* undef
+  store volatile i32 %arg27, i32 addrspace(1)* undef
+  store volatile i32 %arg28, i32 addrspace(1)* undef
+  store volatile i32 %arg29, i32 addrspace(1)* undef
+  store volatile i32 %arg30, i32 addrspace(1)* undef
+
+  ret void
+}
+
+; frame[0] = kernel emergency stack slot
+; frame[1] = callee emergency stack slot
+; frame[2] = ID Y
+; frame[3] = ID Z
+
+; GCN-LABEL: {{^}}kern_call_too_many_args_use_workitem_id_x_stack_yz:
+; GCN: enable_vgpr_workitem_id = 2
+
+; GCN: s_mov_b32 s33, s7
+; GCN: s_mov_b32 s32, s33
+
+; GCN-DAG: v_mov_b32_e32 v31, v0
+; GCN-DAG: buffer_store_dword v1, off, s[0:3], s32 offset:8
+; GCN-DAG: buffer_store_dword v2, off, s[0:3], s32 offset:12
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kern_call_too_many_args_use_workitem_id_x_stack_yz() #1 {
+  call void @too_many_args_use_workitem_id_x_stack_yz(
+    i32 10, i32 20, i32 30, i32 40,
+    i32 50, i32 60, i32 70, i32 80,
+    i32 90, i32 100, i32 110, i32 120,
+    i32 130, i32 140, i32 150, i32 160,
+    i32 170, i32 180, i32 190, i32 200,
+    i32 210, i32 220, i32 230, i32 240,
+    i32 250, i32 260, i32 270, i32 280,
+    i32 290, i32 300, i32 310)
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare i32 @llvm.amdgcn.workitem.id.y() #0
+declare i32 @llvm.amdgcn.workitem.id.z() #0
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/calling-conventions.ll b/test/CodeGen/AMDGPU/calling-conventions.ll
index 677147b6f4e53..5a6fe9ac190b6 100644
--- a/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -43,3 +43,82 @@ define amdgpu_ps half @ps_ret_cc_inreg_f16(half inreg %arg0) {
   %add = fadd half %arg0, 1.0
   ret half %add
 }
+
+; GCN-LABEL: {{^}}fastcc:
+; GCN: v_add_f32_e32 v0, 4.0, v0
+define fastcc float @fastcc(float %arg0) #0 {
+  %add = fadd float %arg0, 4.0
+  ret float %add
+}
+
+; GCN-LABEL: {{^}}coldcc:
+; GCN: v_add_f32_e32 v0, 4.0, v0
+define coldcc float @coldcc(float %arg0) #0 {
+ %add = fadd float %arg0, 4.0
+ ret float %add
+}
+
+; GCN-LABEL: {{^}}call_coldcc:
+; GCN: v_mov_b32_e32 v0, 1.0
+; GCN: s_swappc_b64
+define amdgpu_kernel void @call_coldcc() #0 {
+  %val = call float @coldcc(float 1.0)
+  store float %val, float addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}call_fastcc:
+; GCN: v_mov_b32_e32 v0, 1.0
+; GCN: s_swappc_b64
+define amdgpu_kernel void @call_fastcc() #0 {
+  %val = call float @fastcc(float 1.0)
+  store float %val, float addrspace(1)* undef
+  ret void
+}
+
+; Mesa compute shader: check for 47176 (COMPUTE_PGM_RSRC1) in .AMDGPU.config
+; GCN-LABEL: .AMDGPU.config
+; GCN: .long  47176
+; GCN-LABEL: {{^}}cs_mesa:
+define amdgpu_cs half @cs_mesa(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+; Mesa pixel shader: check for 45096 (SPI_SHADER_PGM_RSRC1_PS) in .AMDGPU.config
+; GCN-LABEL: .AMDGPU.config
+; GCN: .long  45096
+; GCN-LABEL: {{^}}ps_mesa:
+define amdgpu_ps half @ps_mesa(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+; Mesa vertex shader: check for 45352 (SPI_SHADER_PGM_RSRC1_VS) in .AMDGPU.config
+; GCN-LABEL: .AMDGPU.config
+; GCN: .long  45352
+; GCN-LABEL: {{^}}vs_mesa:
+define amdgpu_vs half @vs_mesa(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+; Mesa geometry shader: check for 45608 (SPI_SHADER_PGM_RSRC1_GS) in .AMDGPU.config
+; GCN-LABEL: .AMDGPU.config
+; GCN: .long  45608
+; GCN-LABEL: {{^}}gs_mesa:
+define amdgpu_gs half @gs_mesa(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+; Mesa hull shader: check for 46120 (SPI_SHADER_PGM_RSRC1_HS) in .AMDGPU.config
+; GCN-LABEL: .AMDGPU.config
+; GCN: .long  46120
+; GCN-LABEL: {{^}}hs_mesa:
+define amdgpu_hs half @hs_mesa(half %arg0) {
+  %add = fadd half %arg0, 1.0
+  ret half %add
+}
+
+attributes #0 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/captured-frame-index.ll b/test/CodeGen/AMDGPU/captured-frame-index.ll
index 5fe1b2728506e..99f6ec48be735 100644
--- a/test/CodeGen/AMDGPU/captured-frame-index.ll
+++ b/test/CodeGen/AMDGPU/captured-frame-index.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-- -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tahiti -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}store_fi_lifetime:
 ; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 4{{$}}
diff --git a/test/CodeGen/AMDGPU/cf-loop-on-constant.ll b/test/CodeGen/AMDGPU/cf-loop-on-constant.ll
index 697f26b83a4df..1e0af2611b022 100644
--- a/test/CodeGen/AMDGPU/cf-loop-on-constant.ll
+++ b/test/CodeGen/AMDGPU/cf-loop-on-constant.ll
@@ -102,7 +102,7 @@ for.body:
 ; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 4
 
 ; GCN: s_cbranch_vccnz [[LOOPBB]]
-; GCN-NEXT: ; BB#2
+; GCN-NEXT: ; %bb.2
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @loop_arg_0(float addrspace(3)* %ptr, i32 %n, i1 %cond) nounwind {
 entry:
diff --git a/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
index c01d834bc33d6..beb0723915799 100644
--- a/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
+++ b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll
@@ -1,9 +1,11 @@
-; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI %s
-; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI %s
-; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-scalarize-global-loads=false -mattr=-flat-for-global -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tahiti < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-SI -check-prefix=OPT-SICIVI %s
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=bonaire < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-CI -check-prefix=OPT-SICIVI %s
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-VI -check-prefix=OPT-SICIVI %s
+; RUN: opt -S -codegenprepare -mtriple=amdgcn-unknown-unknown -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT -check-prefix=OPT-GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-scalarize-global-loads=false -mattr=-promote-alloca -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-scalarize-global-loads=false -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
 
@@ -14,7 +16,6 @@ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:
 ; OPT-CI: getelementptr i8,
 
 ; GCN-LABEL: {{^}}test_sink_global_small_offset_i32:
-; GCN: {{^}}BB0_2:
 define amdgpu_kernel void @test_sink_global_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
 entry:
   %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 999999
@@ -42,7 +43,8 @@ done:
 
 ; GCN-LABEL: {{^}}test_sink_global_small_max_i32_ds_offset:
 ; GCN: s_and_saveexec_b64
-; GCN: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
 ; GCN: {{^}}BB1_2:
 ; GCN: s_or_b64 exec
 define amdgpu_kernel void @test_sink_global_small_max_i32_ds_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
@@ -69,7 +71,8 @@ done:
 
 ; GCN-LABEL: {{^}}test_sink_global_small_max_mubuf_offset:
 ; GCN: s_and_saveexec_b64
-; GCN: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4095{{$}}
+; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:4095{{$}}
+; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off offset:4095{{$}}
 ; GCN: {{^}}BB2_2:
 ; GCN: s_or_b64 exec
 define amdgpu_kernel void @test_sink_global_small_max_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
@@ -96,7 +99,8 @@ done:
 
 ; GCN-LABEL: {{^}}test_sink_global_small_max_plus_1_mubuf_offset:
 ; GCN: s_and_saveexec_b64
-; GCN: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; SICIVI: buffer_load_sbyte {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
+; GFX9: global_load_sbyte {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
 ; GCN: {{^}}BB3_2:
 ; GCN: s_or_b64 exec
 define amdgpu_kernel void @test_sink_global_small_max_plus_1_mubuf_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
@@ -673,6 +677,67 @@ done:
   ret void
 }
 
+; OPT-LABEL: @test_sink_global_small_min_scratch_global_offset(
+; OPT-SICIVI: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4096
+; OPT-SICIV: br
+; OPT-SICIVI: %tmp1 = load i8, i8 addrspace(1)* %in.gep
+
+; OPT-GFX9: br
+; OPT-GFX9: %sunkaddr = getelementptr i8, i8 addrspace(1)* %in, i64 -4096
+; OPT-GFX9: load i8, i8 addrspace(1)* %sunkaddr
+
+; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_offset:
+; GFX9: global_load_sbyte v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:-4096{{$}}
+define amdgpu_kernel void @test_sink_global_small_min_scratch_global_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024
+  %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4096
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %tmp0 = icmp eq i32 %tid, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i8, i8 addrspace(1)* %in.gep
+  %tmp2 = sext i8 %tmp1 to i32
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
+; OPT-LABEL: @test_sink_global_small_min_scratch_global_neg1_offset(
+; OPT: %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4097
+; OPT: br
+; OPT: load i8, i8 addrspace(1)* %in.gep
+
+; GCN-LABEL: {{^}}test_sink_global_small_min_scratch_global_neg1_offset:
+define amdgpu_kernel void @test_sink_global_small_min_scratch_global_neg1_offset(i32 addrspace(1)* %out, i8 addrspace(1)* %in) {
+entry:
+  %out.gep = getelementptr i32, i32 addrspace(1)* %out, i64 99999
+  %in.gep = getelementptr i8, i8 addrspace(1)* %in, i64 -4097
+  %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
+  %tmp0 = icmp eq i32 %tid, 0
+  br i1 %tmp0, label %endif, label %if
+
+if:
+  %tmp1 = load i8, i8 addrspace(1)* %in.gep
+  %tmp2 = sext i8 %tmp1 to i32
+  br label %endif
+
+endif:
+  %x = phi i32 [ %tmp2, %if ], [ 0, %entry ]
+  store i32 %x, i32 addrspace(1)* %out.gep
+  br label %done
+
+done:
+  ret void
+}
+
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
 declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
diff --git a/test/CodeGen/AMDGPU/clamp-modifier.ll b/test/CodeGen/AMDGPU/clamp-modifier.ll
index c3a7d5e14d87c..9c137101c56d1 100644
--- a/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1,8 +1,9 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN-NOT: [[A]]
 ; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
 define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -18,7 +19,7 @@ define amdgpu_kernel void @v_clamp_add_src_f32(float addrspace(1)* %out, float a
 }
 
 ; GCN-LABEL: {{^}}v_clamp_multi_use_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[ADD]], [[ADD]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -35,7 +36,7 @@ define amdgpu_kernel void @v_clamp_multi_use_src_f32(float addrspace(1)* %out, f
 }
 
 ; GCN-LABEL: {{^}}v_clamp_dbg_use_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN-NOT: [[A]]
 ; GCN: v_add_f32_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
 define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -52,7 +53,7 @@ define amdgpu_kernel void @v_clamp_dbg_use_src_f32(float addrspace(1)* %out, flo
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_neg_src_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_floor_f32_e32 [[FLOOR:v[0-9]+]], [[A]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, -[[FLOOR]], -[[FLOOR]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -69,7 +70,7 @@ define amdgpu_kernel void @v_clamp_add_neg_src_f32(float addrspace(1)* %out, flo
 }
 
 ; GCN-LABEL: {{^}}v_non_clamp_max_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
 ; GCN: v_max_f32_e32 v{{[0-9]+}}, 0, [[ADD]]{{$}}
 define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -84,7 +85,7 @@ define amdgpu_kernel void @v_non_clamp_max_f32(float addrspace(1)* %out, float a
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_f32_denormals:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_add_f32_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
 define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -99,8 +100,8 @@ define amdgpu_kernel void @v_clamp_add_src_f32_denormals(float addrspace(1)* %ou
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_f16_denorm:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89: v_add_f16_e64 [[ADD:v[0-9]+]], [[A]], 1.0 clamp{{$}}
 
 ; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
 ; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
@@ -118,9 +119,9 @@ define amdgpu_kernel void @v_clamp_add_src_f16_denorm(half addrspace(1)* %out, h
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_f16_no_denormals:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI-NOT: [[A]]
-; VI: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89-NOT: [[A]]
+; GFX89: v_add_f16_e64 v{{[0-9]+}}, [[A]], 1.0 clamp{{$}}
 
 ; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
 ; SI: v_add_f32_e64 [[ADD:v[0-9]+]], [[CVT]], 1.0 clamp{{$}}
@@ -138,7 +139,7 @@ define amdgpu_kernel void @v_clamp_add_src_f16_no_denormals(half addrspace(1)* %
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_v2f32:
-; GCN: {{buffer|flat}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
+; GCN: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[A:[0-9]+]]:[[B:[0-9]+]]{{\]}}
 ; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[A]], 1.0 clamp{{$}}
 ; GCN-DAG: v_add_f32_e64 v{{[0-9]+}}, v[[B]], 1.0 clamp{{$}}
 define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out, <2 x float> addrspace(1)* %aptr) #0 {
@@ -154,7 +155,7 @@ define amdgpu_kernel void @v_clamp_add_src_v2f32(<2 x float> addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}v_clamp_add_src_f64:
-; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
 ; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], 1.0 clamp{{$}}
 define amdgpu_kernel void @v_clamp_add_src_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -185,6 +186,173 @@ define amdgpu_kernel void @v_clamp_mac_to_mad(float addrspace(1)* %out, float ad
   ret void
 }
 
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_no_denormals:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_no_denormals(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #3 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %add, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %neg.add = fsub <2 x half> <half -0.0, half -0.0>, %add
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.add, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_lo:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_lo:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_lo(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %lo = extractelement <2 x half> %add, i32 0
+  %neg.lo = fsub half -0.0, %lo
+  %neg.lo.add = insertelement <2 x half> %add, half %neg.lo, i32 0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.lo.add, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_neg_hi:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_neg_hi(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %hi = extractelement <2 x half> %add, i32 1
+  %neg.hi = fsub half -0.0, %hi
+  %neg.hi.add = insertelement <2 x half> %add, half %neg.hi, i32 1
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.hi.add, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_add_src_v2f16_denorm_shuf:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_pk_max_f16 [[MAX:v[0-9]+]], [[ADD]], [[ADD]] op_sel:[1,1] op_sel_hi:[0,0] clamp{{$}}
+define amdgpu_kernel void @v_clamp_add_src_v2f16_denorm_shuf(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %shuf = shufflevector <2 x half> %add, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f32_src:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f32_src(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %bc = bitcast <2 x half> %a to float
+  %f32.op = fadd float %bc, 1.0
+  %f32.op.cast = bitcast float %f32.op to <2 x half>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %f32.op.cast, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_no_clamp_add_packed_src_f32:
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-DAG: s_mov_b32 [[ONE:s[0-9]+]], 0x3c003c00
+; GFX9: v_pk_add_f16 [[ADD:v[0-9]+]], [[A]], [[ONE]]{{$}}
+; GFX9: v_max_f32_e64 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_packed_src_f32(float addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr float, float addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %add = fadd <2 x half> %a, <half 1.0, half 1.0>
+  %bc.add = bitcast <2 x half> %add to float
+  %max = call float @llvm.maxnum.f32(float %bc.add, float 0.0)
+  %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+  store float %clamp, float addrspace(1)* %out.gep
+  ret void
+}
+
+; Since the high bits are zeroed, it probably would be OK in this case
+; to use clamp.
+; GCN-LABEL: {{^}}v_no_clamp_add_src_v2f16_f16_src:
+; GCN-DAG: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX9: v_add_f16_e32 [[ADD:v[0-9]+]], 1.0, [[A]]{{$}}
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ADD]], [[ADD]] clamp{{$}}
+define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(<2 x half> addrspace(1)* %out, half addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr half, half addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load half, half addrspace(1)* %gep0
+  %add = fadd half %a, 1.0
+  %bc = bitcast half %add to i16
+  %zext = zext i16 %bc to i32
+  %v2f16 = bitcast i32 %zext to <2 x half>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %v2f16, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  store <2 x half> %clamp, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare float @llvm.fabs.f32(float) #1
 declare float @llvm.floor.f32(float) #1
@@ -197,8 +365,12 @@ declare double @llvm.maxnum.f64(double, double) #1
 declare half @llvm.fabs.f16(half) #1
 declare half @llvm.minnum.f16(half, half) #1
 declare half @llvm.maxnum.f16(half, half) #1
+declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
+declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
 declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1
 declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1
+
+
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/clamp-omod-special-case.mir b/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
index 90fba03420901..8ab99c6d2969d 100644
--- a/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
+++ b/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
@@ -1,8 +1,8 @@
 # RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-fold-operands  %s -o - | FileCheck -check-prefix=GCN %s
 ---
 # GCN-LABEL: name: v_max_self_clamp_not_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN-NEXT: %21 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 0, 0, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN-NEXT: %21:vgpr_32 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 0, 0, implicit %exec
 
 name:            v_max_self_clamp_not_set_f32
 tracksRegLiveness: true
@@ -64,8 +64,8 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: v_clamp_omod_already_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN: %21 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 1, 3, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN: %21:vgpr_32 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 1, 3, implicit %exec
 name:            v_clamp_omod_already_set_f32
 tracksRegLiveness: true
 registers:
@@ -127,8 +127,8 @@ body:             |
 # Don't fold a mul that looks like an omod if itself has omod set
 
 # GCN-LABEL: name: v_omod_mul_omod_already_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN-NEXT: %21 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 0, 3, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN-NEXT: %21:vgpr_32 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 0, 3, implicit %exec
 name:            v_omod_mul_omod_already_set_f32
 tracksRegLiveness: true
 registers:
@@ -191,8 +191,8 @@ body:             |
 # Don't fold a mul that looks like an omod if itself has clamp set
 # This might be OK, but would require folding the clamp at the same time.
 # GCN-LABEL: name: v_omod_mul_clamp_already_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN-NEXT: %21 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 1, 0, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN-NEXT: %21:vgpr_32 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 1, 0, implicit %exec
 
 name:            v_omod_mul_clamp_already_set_f32
 tracksRegLiveness: true
@@ -269,8 +269,8 @@ body:             |
 # Don't fold a mul that looks like an omod if itself has omod set
 
 # GCN-LABEL: name: v_omod_add_omod_already_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN-NEXT: %21 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 0, 3, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN-NEXT: %21:vgpr_32 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 0, 3, implicit %exec
 name:            v_omod_add_omod_already_set_f32
 tracksRegLiveness: true
 registers:
@@ -333,8 +333,8 @@ body:             |
 # Don't fold a mul that looks like an omod if itself has clamp set
 # This might be OK, but would require folding the clamp at the same time.
 # GCN-LABEL: name: v_omod_add_clamp_already_set_f32
-# GCN: %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
-# GCN-NEXT: %21 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 1, 0, implicit %exec
+# GCN: %20:vgpr_32 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit %exec
+# GCN-NEXT: %21:vgpr_32 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 1, 0, implicit %exec
 
 name:            v_omod_add_clamp_already_set_f32
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/clamp.ll b/test/CodeGen/AMDGPU/clamp.ll
index 9735c7074be2a..683235764caf2 100644
--- a/test/CodeGen/AMDGPU/clamp.ll
+++ b/test/CodeGen/AMDGPU/clamp.ll
@@ -1,8 +1,9 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
 
 ; GCN-LABEL: {{^}}v_clamp_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -17,7 +18,7 @@ define amdgpu_kernel void @v_clamp_f32(float addrspace(1)* %out, float addrspace
 }
 
 ; GCN-LABEL: {{^}}v_clamp_neg_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, -[[A]], -[[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_neg_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -33,7 +34,7 @@ define amdgpu_kernel void @v_clamp_neg_f32(float addrspace(1)* %out, float addrs
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negabs_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, -|[[A]]|, -|[[A]]| clamp{{$}}
 define amdgpu_kernel void @v_clamp_negabs_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -51,7 +52,7 @@ define amdgpu_kernel void @v_clamp_negabs_f32(float addrspace(1)* %out, float ad
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negzero_f32:
-; GCN-DAG: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN-DAG: v_bfrev_b32_e32 [[SIGNBIT:v[0-9]+]], 1
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[SIGNBIT]], 1.0
 define amdgpu_kernel void @v_clamp_negzero_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -67,7 +68,7 @@ define amdgpu_kernel void @v_clamp_negzero_f32(float addrspace(1)* %out, float a
 }
 
 ; GCN-LABEL: {{^}}v_clamp_multi_use_max_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e32 [[MAX:v[0-9]+]], 0, [[A]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], 1.0, [[MAX]]
 define amdgpu_kernel void @v_clamp_multi_use_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
@@ -84,8 +85,8 @@ define amdgpu_kernel void @v_clamp_multi_use_max_f32(float addrspace(1)* %out, f
 }
 
 ; GCN-LABEL: {{^}}v_clamp_f16:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI: v_max_f16_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89: v_max_f16_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 
 ; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], [[A]] clamp{{$}}
 ; SI: v_cvt_f16_f32_e32 v{{[0-9]+}}, [[CVT]]
@@ -102,8 +103,8 @@ define amdgpu_kernel void @v_clamp_f16(half addrspace(1)* %out, half addrspace(1
 }
 
 ; GCN-LABEL: {{^}}v_clamp_neg_f16:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI: v_max_f16_e64 v{{[0-9]+}}, -[[A]], -[[A]] clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89: v_max_f16_e64 v{{[0-9]+}}, -[[A]], -[[A]] clamp{{$}}
 
 ; FIXME: Better to fold neg into max
 ; SI: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -[[A]] clamp{{$}}
@@ -122,8 +123,8 @@ define amdgpu_kernel void @v_clamp_neg_f16(half addrspace(1)* %out, half addrspa
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negabs_f16:
-; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]
-; VI: v_max_f16_e64 v{{[0-9]+}}, -|[[A]]|, -|[[A]]| clamp{{$}}
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GFX89: v_max_f16_e64 v{{[0-9]+}}, -|[[A]]|, -|[[A]]| clamp{{$}}
 
 ; FIXME: Better to fold neg/abs into max
 
@@ -146,7 +147,7 @@ define amdgpu_kernel void @v_clamp_negabs_f16(half addrspace(1)* %out, half addr
 
 ; FIXME: Do f64 instructions support clamp?
 ; GCN-LABEL: {{^}}v_clamp_f64:
-; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
 ; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -161,7 +162,7 @@ define amdgpu_kernel void @v_clamp_f64(double addrspace(1)* %out, double addrspa
 }
 
 ; GCN-LABEL: {{^}}v_clamp_neg_f64:
-; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
 ; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, -[[A]], -[[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_neg_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -177,7 +178,7 @@ define amdgpu_kernel void @v_clamp_neg_f64(double addrspace(1)* %out, double add
 }
 
 ; GCN-LABEL: {{^}}v_clamp_negabs_f64:
-; GCN: {{buffer|flat}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
+; GCN: {{buffer|flat|global}}_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]]
 ; GCN: v_max_f64 v{{\[[0-9]+:[0-9]+\]}}, -|[[A]]|, -|[[A]]| clamp{{$}}
 define amdgpu_kernel void @v_clamp_negabs_f64(double addrspace(1)* %out, double addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -195,7 +196,7 @@ define amdgpu_kernel void @v_clamp_negabs_f64(double addrspace(1)* %out, double
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_aby_negzero_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32
 define amdgpu_kernel void @v_clamp_med3_aby_negzero_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -208,7 +209,7 @@ define amdgpu_kernel void @v_clamp_med3_aby_negzero_f32(float addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_aby_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_aby_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -221,7 +222,7 @@ define amdgpu_kernel void @v_clamp_med3_aby_f32(float addrspace(1)* %out, float
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_bay_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_bay_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -234,7 +235,7 @@ define amdgpu_kernel void @v_clamp_med3_bay_f32(float addrspace(1)* %out, float
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_yab_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_yab_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -247,7 +248,7 @@ define amdgpu_kernel void @v_clamp_med3_yab_f32(float addrspace(1)* %out, float
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_yba_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_yba_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -260,7 +261,7 @@ define amdgpu_kernel void @v_clamp_med3_yba_f32(float addrspace(1)* %out, float
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_ayb_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_ayb_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -273,7 +274,7 @@ define amdgpu_kernel void @v_clamp_med3_ayb_f32(float addrspace(1)* %out, float
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_bya_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_bya_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -350,7 +351,7 @@ define amdgpu_kernel void @v_clamp_constant_snan_f32(float addrspace(1)* %out) #
 ; ---------------------------------------------------------------------
 
 ; GCN-LABEL: {{^}}v_clamp_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], 0, 1.0
 define amdgpu_kernel void @v_clamp_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -365,7 +366,7 @@ define amdgpu_kernel void @v_clamp_f32_no_dx10_clamp(float addrspace(1)* %out, f
 }
 
 ; GCN-LABEL: {{^}}v_clamp_f32_snan_dx10clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_f32_snan_dx10clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #3 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -380,7 +381,7 @@ define amdgpu_kernel void @v_clamp_f32_snan_dx10clamp(float addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}v_clamp_f32_snan_no_dx10clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e32 [[MAX:v[0-9]+]], 0, [[A]]
 ; GCN: v_min_f32_e32 [[MIN:v[0-9]+]], 1.0, [[MAX]]
 define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #4 {
@@ -396,8 +397,9 @@ define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp(float addrspace(1)* %ou
 }
 
 ; GCN-LABEL: {{^}}v_clamp_f32_snan_no_dx10clamp_nnan_src:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], 0, 1.0
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, [[A]]
+; GCN: v_med3_f32 v{{[0-9]+}}, [[ADD]], 0, 1.0
 define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp_nnan_src(float addrspace(1)* %out, float addrspace(1)* %aptr) #4 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, float addrspace(1)* %aptr, i32 %tid
@@ -412,7 +414,7 @@ define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp_nnan_src(float addrspac
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_aby_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_aby_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -425,7 +427,7 @@ define amdgpu_kernel void @v_clamp_med3_aby_f32_no_dx10_clamp(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_bay_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[A]] clamp{{$}}
 define amdgpu_kernel void @v_clamp_med3_bay_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -438,7 +440,7 @@ define amdgpu_kernel void @v_clamp_med3_bay_f32_no_dx10_clamp(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_yab_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], 0, 1.0
 define amdgpu_kernel void @v_clamp_med3_yab_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -451,7 +453,7 @@ define amdgpu_kernel void @v_clamp_med3_yab_f32_no_dx10_clamp(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_yba_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], 1.0, 0
 define amdgpu_kernel void @v_clamp_med3_yba_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -464,7 +466,7 @@ define amdgpu_kernel void @v_clamp_med3_yba_f32_no_dx10_clamp(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_ayb_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, 0, [[A]], 1.0
 define amdgpu_kernel void @v_clamp_med3_ayb_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -477,7 +479,7 @@ define amdgpu_kernel void @v_clamp_med3_ayb_f32_no_dx10_clamp(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_clamp_med3_bya_f32_no_dx10_clamp:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, 1.0, [[A]], 0
 define amdgpu_kernel void @v_clamp_med3_bya_f32_no_dx10_clamp(float addrspace(1)* %out, float addrspace(1)* %aptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -509,6 +511,181 @@ define amdgpu_kernel void @v_clamp_constant_snan_f32_no_dx10_clamp(float addrspa
   ret void
 }
 
+; GCN-LABEL: {{^}}v_clamp_v2f16:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_v2f16_undef_elt:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_v2f16_undef_elt(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> <half undef, half 0.0>)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half undef>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_v2f16_not_zero:
+; GFX9: v_pk_max_f16
+; GFX9: v_pk_min_f16
+define amdgpu_kernel void @v_clamp_v2f16_not_zero(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> <half 2.0, half 0.0>)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_v2f16_not_one:
+; GFX9: v_pk_max_f16
+; GFX9: v_pk_min_f16
+define amdgpu_kernel void @v_clamp_v2f16_not_one(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> <half 0.0, half 0.0>)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 0.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_neg_v2f16:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_neg_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %fneg.a = fsub <2 x half> <half -0.0, half -0.0>, %a
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %fneg.a, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_negabs_v2f16:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9: v_and_b32_e32 [[ABS:v[0-9]+]], 0x7fff7fff, [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[ABS]], [[ABS]] neg_lo:[1,1] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_negabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %fabs.a = call <2 x half> @llvm.fabs.v2f16(<2 x half> %a)
+  %fneg.fabs.a = fsub <2 x half> <half -0.0, half -0.0>, %fabs.a
+
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %fneg.fabs.a, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_neglo_v2f16:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] neg_lo:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_neglo_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %lo = extractelement <2 x half> %a, i32 0
+  %neg.lo = fsub half -0.0, %lo
+  %neg.lo.vec = insertelement <2 x half> %a, half %neg.lo, i32 0
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.lo.vec, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_neghi_v2f16:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] neg_hi:[1,1] clamp{{$}}
+define amdgpu_kernel void @v_clamp_neghi_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %hi = extractelement <2 x half> %a, i32 1
+  %neg.hi = fsub half -0.0, %hi
+  %neg.hi.vec = insertelement <2 x half> %a, half %neg.hi, i32 1
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %neg.hi.vec, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_v2f16_shuffle:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GFX9-NOT: [[A]]
+; GFX9: v_pk_max_f16 [[CLAMP:v[0-9]+]], [[A]], [[A]] op_sel:[1,1] op_sel_hi:[0,0] clamp{{$}}
+define amdgpu_kernel void @v_clamp_v2f16_shuffle(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %aptr) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %aptr, i32 %tid
+  %out.gep = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
+  %a = load <2 x half>, <2 x half> addrspace(1)* %gep0
+  %shuf = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %shuf, <2 x half> zeroinitializer)
+  %med = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+
+  store <2 x half> %med, <2 x half> addrspace(1)* %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_clamp_diff_source_f32:
+; GCN: v_add_f32_e32 [[A:v[0-9]+]]
+; GCN: v_add_f32_e32 [[B:v[0-9]+]]
+; GCN: v_max_f32_e64 v{{[0-9]+}}, [[A]], [[B]] clamp{{$}}
+define amdgpu_kernel void @v_clamp_diff_source_f32(float addrspace(1)* %out, float addrspace(1)* %aptr) #0
+{
+  %gep0 = getelementptr float, float addrspace(1)* %aptr, i32 0
+  %gep1 = getelementptr float, float addrspace(1)* %aptr, i32 1
+  %gep2 = getelementptr float, float addrspace(1)* %aptr, i32 2
+  %l0 = load float, float addrspace(1)* %gep0
+  %l1 = load float, float addrspace(1)* %gep1
+  %l2 = load float, float addrspace(1)* %gep2
+  %a = fadd nsz float %l0, %l1
+  %b = fadd nsz float %l0, %l2
+  %res = call nsz float @llvm.maxnum.f32(float %a, float %b)
+  %max = call nsz float @llvm.maxnum.f32(float %res, float 0.0)
+  %min = call nsz float @llvm.minnum.f32(float %max, float 1.0)
+  %out.gep = getelementptr float, float addrspace(1)* %out, i32 3
+  store float %min, float addrspace(1)* %out.gep
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare float @llvm.fabs.f32(float) #1
 declare float @llvm.minnum.f32(float, float) #1
@@ -520,7 +697,9 @@ declare double @llvm.maxnum.f64(double, double) #1
 declare half @llvm.fabs.f16(half) #1
 declare half @llvm.minnum.f16(half, half) #1
 declare half @llvm.maxnum.f16(half, half) #1
-
+declare <2 x half> @llvm.fabs.v2f16(<2 x half>) #1
+declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
+declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir b/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
new file mode 100644
index 0000000000000..4c528b5783357
--- /dev/null
+++ b/test/CodeGen/AMDGPU/cluster-flat-loads-postra.mir
@@ -0,0 +1,31 @@
+# RUN: llc -march=amdgcn -mcpu=tonga -run-pass post-RA-sched -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+
+# GCN:      FLAT_LOAD_DWORD
+# GCN-NEXT: FLAT_LOAD_DWORD
+# GCN:      FLAT_STORE_DWORD
+# GCN-NEXT: FLAT_STORE_DWORD
+
+---
+name:            cluster_loads_post_ra
+tracksRegLiveness: true
+registers:
+liveins:
+  - { reg: '%vgpr0' }
+body:             |
+  bb.0:
+    liveins: %vgpr0
+
+    %vgpr0_vgpr1 = IMPLICIT_DEF
+    %vgpr4_vgpr5 = IMPLICIT_DEF
+    %vgpr0 = FLAT_LOAD_DWORD %vgpr0_vgpr1, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+    %vgpr4 = FLAT_LOAD_DWORD %vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+    %vgpr2 = IMPLICIT_DEF
+    %vgpr3 = IMPLICIT_DEF
+    %vgpr6 = IMPLICIT_DEF
+    %vgpr0 = V_ADD_I32_e32 16, %vgpr2, implicit-def %vcc, implicit %exec
+    %vgpr1 = V_ADDC_U32_e32 %vgpr3, killed %vgpr6, implicit-def dead %vcc, implicit %vcc, implicit %exec
+    FLAT_STORE_DWORD %vgpr2_vgpr3, killed %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4)
+    FLAT_STORE_DWORD %vgpr0_vgpr1, killed %vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4)
+    S_ENDPGM
+
+...
diff --git a/test/CodeGen/AMDGPU/cluster-flat-loads.mir b/test/CodeGen/AMDGPU/cluster-flat-loads.mir
new file mode 100644
index 0000000000000..50caba213d3a0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/cluster-flat-loads.mir
@@ -0,0 +1,20 @@
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass machine-scheduler %s -o - | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: cluster_flat_loads
+# GCN:      FLAT_LOAD_DWORD %0, 0
+# GCN-NEXT: FLAT_LOAD_DWORD %0, 4
+# GCN-NEXT: V_ADD_F32_e64
+name: cluster_flat_loads
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+    %0 = IMPLICIT_DEF
+    %1 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+    %2 = V_ADD_F32_e64 0, killed %1, 0, 1, 0, 0, implicit %exec
+    %3 = FLAT_LOAD_DWORD %0, 4, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+...
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-kernel-code-props.ll b/test/CodeGen/AMDGPU/code-object-metadata-kernel-code-props.ll
deleted file mode 100644
index 3b232e40cf25c..0000000000000
--- a/test/CodeGen/AMDGPU/code-object-metadata-kernel-code-props.ll
+++ /dev/null
@@ -1,32 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
-
-; CHECK: ---
-; CHECK:  Version: [ 1, 0 ]
-
-; CHECK:  Kernels:
-; CHECK:    - Name: test
-; CHECK:      CodeProps:
-; CHECK:        KernargSegmentSize:  24
-; GFX700:       WavefrontNumSGPRs:   6
-; GFX800:       WavefrontNumSGPRs:   96
-; GFX900:       WavefrontNumSGPRs:   6
-; GFX700:       WorkitemNumVGPRs:    4
-; GFX800:       WorkitemNumVGPRs:    6
-; GFX900:       WorkitemNumVGPRs:    6
-; CHECK:        KernargSegmentAlign: 4
-; CHECK:        GroupSegmentAlign:   4
-; CHECK:        PrivateSegmentAlign: 4
-; CHECK:        WavefrontSize:       6
-define amdgpu_kernel void @test(
-    half addrspace(1)* %r,
-    half addrspace(1)* %a,
-    half addrspace(1)* %b) {
-entry:
-  %a.val = load half, half addrspace(1)* %a
-  %b.val = load half, half addrspace(1)* %b
-  %r.val = fadd half %a.val, %b.val
-  store half %r.val, half addrspace(1)* %r
-  ret void
-}
diff --git a/test/CodeGen/AMDGPU/collapse-endcf.ll b/test/CodeGen/AMDGPU/collapse-endcf.ll
new file mode 100644
index 0000000000000..f359ffc284a52
--- /dev/null
+++ b/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -0,0 +1,269 @@
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}simple_nested_if:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9_]+]]
+; GCN-NEXT: s_cbranch_execz [[ENDIF]]
+; GCN:      s_and_b64 exec, exec, vcc
+; GCN-NEXT: ; mask branch [[ENDIF]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF]]:
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @simple_nested_if(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = icmp ugt i32 %tmp, 1
+  br i1 %tmp1, label %bb.outer.then, label %bb.outer.end
+
+bb.outer.then:                                    ; preds = %bb
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
+  store i32 0, i32 addrspace(1)* %tmp4, align 4
+  %tmp5 = icmp eq i32 %tmp, 2
+  br i1 %tmp5, label %bb.outer.end, label %bb.inner.then
+
+bb.inner.then:                                    ; preds = %bb.outer.then
+  %tmp7 = add i32 %tmp, 1
+  %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp7
+  store i32 1, i32 addrspace(1)* %tmp9, align 4
+  br label %bb.outer.end
+
+bb.outer.end:                                     ; preds = %bb.outer.then, %bb.inner.then, %bb
+  ret void
+}
+
+; GCN-LABEL: {{^}}uncollapsable_nested_if:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF_OUTER:BB[0-9_]+]]
+; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER]]
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC_INNER:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF_INNER:BB[0-9_]+]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF_INNER]]:
+; GCN-NEXT: s_or_b64 exec, exec, [[SAVEEXEC_INNER]]
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF_OUTER]]:
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @uncollapsable_nested_if(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = icmp ugt i32 %tmp, 1
+  br i1 %tmp1, label %bb.outer.then, label %bb.outer.end
+
+bb.outer.then:                                    ; preds = %bb
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
+  store i32 0, i32 addrspace(1)* %tmp4, align 4
+  %tmp5 = icmp eq i32 %tmp, 2
+  br i1 %tmp5, label %bb.inner.end, label %bb.inner.then
+
+bb.inner.then:                                    ; preds = %bb.outer.then
+  %tmp7 = add i32 %tmp, 1
+  %tmp8 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp7
+  store i32 1, i32 addrspace(1)* %tmp8, align 4
+  br label %bb.inner.end
+
+bb.inner.end:                                     ; preds = %bb.inner.then, %bb.outer.then
+  %tmp9 = add i32 %tmp, 2
+  %tmp10 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp9
+  store i32 2, i32 addrspace(1)* %tmp10, align 4
+  br label %bb.outer.end
+
+bb.outer.end:                                     ; preds = %bb.inner.then, %bb
+  ret void
+}
+
+; GCN-LABEL: {{^}}nested_if_if_else:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF_OUTER:BB[0-9_]+]]
+; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER]]
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC_INNER:s\[[0-9:]+\]]]
+; GCN-NEXT: s_xor_b64 [[SAVEEXEC_INNER2:s\[[0-9:]+\]]], exec, [[SAVEEXEC_INNER]]
+; GCN-NEXT: ; mask branch [[THEN_INNER:BB[0-9_]+]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[THEN_INNER]]:
+; GCN-NEXT: s_or_saveexec_b64 [[SAVEEXEC_INNER3:s\[[0-9:]+\]]], [[SAVEEXEC_INNER2]]
+; GCN-NEXT: s_xor_b64 exec, exec, [[SAVEEXEC_INNER3]]
+; GCN-NEXT: ; mask branch [[ENDIF_OUTER]]
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF_OUTER]]:
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @nested_if_if_else(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
+  store i32 0, i32 addrspace(1)* %tmp1, align 4
+  %tmp2 = icmp ugt i32 %tmp, 1
+  br i1 %tmp2, label %bb.outer.then, label %bb.outer.end
+
+bb.outer.then:                                       ; preds = %bb
+  %tmp5 = icmp eq i32 %tmp, 2
+  br i1 %tmp5, label %bb.then, label %bb.else
+
+bb.then:                                             ; preds = %bb.outer.then
+  %tmp3 = add i32 %tmp, 1
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp3
+  store i32 1, i32 addrspace(1)* %tmp4, align 4
+  br label %bb.outer.end
+
+bb.else:                                             ; preds = %bb.outer.then
+  %tmp7 = add i32 %tmp, 2
+  %tmp9 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp7
+  store i32 2, i32 addrspace(1)* %tmp9, align 4
+  br label %bb.outer.end
+
+bb.outer.end:                                        ; preds = %bb, %bb.then, %bb.else
+  ret void
+}
+
+; GCN-LABEL: {{^}}nested_if_else_if:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC_OUTER:s\[[0-9:]+\]]]
+; GCN-NEXT: s_xor_b64 [[SAVEEXEC_OUTER2:s\[[0-9:]+\]]], exec, [[SAVEEXEC_OUTER]]
+; GCN-NEXT: ; mask branch [[THEN_OUTER:BB[0-9_]+]]
+; GCN-NEXT: s_cbranch_execz [[THEN_OUTER]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: s_and_saveexec_b64 [[SAVEEXEC_INNER_IF_OUTER_ELSE:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[THEN_OUTER_FLOW:BB[0-9_]+]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[THEN_OUTER_FLOW]]:
+; GCN-NEXT: s_or_b64 exec, exec, [[SAVEEXEC_INNER_IF_OUTER_ELSE]]
+; GCN-NEXT: {{^}}[[THEN_OUTER]]:
+; GCN-NEXT: s_or_saveexec_b64 [[SAVEEXEC_OUTER3:s\[[0-9:]+\]]], [[SAVEEXEC_OUTER2]]
+; GCN-NEXT: s_xor_b64 exec, exec, [[SAVEEXEC_OUTER3]]
+; GCN-NEXT: ; mask branch [[ENDIF_OUTER:BB[0-9_]+]]
+; GCN-NEXT: s_cbranch_execz [[ENDIF_OUTER]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: s_and_saveexec_b64 [[SAVEEXEC_INNER_IF_OUTER_THEN:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF_OUTER]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF_OUTER]]:
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @nested_if_else_if(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
+  store i32 0, i32 addrspace(1)* %tmp1, align 4
+  %cc1 = icmp ugt i32 %tmp, 1
+  br i1 %cc1, label %bb.outer.then, label %bb.outer.else
+
+bb.outer.then:
+  %tmp2 = getelementptr inbounds i32, i32 addrspace(1)* %tmp1, i32 1
+  store i32 1, i32 addrspace(1)* %tmp2, align 4
+  %cc2 = icmp eq i32 %tmp, 2
+  br i1 %cc2, label %bb.inner.then, label %bb.outer.end
+
+bb.inner.then:
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %tmp1, i32 2
+  store i32 2, i32 addrspace(1)* %tmp3, align 4
+  br label %bb.outer.end
+
+bb.outer.else:
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %tmp1, i32 3
+  store i32 3, i32 addrspace(1)* %tmp4, align 4
+  %cc3 = icmp eq i32 %tmp, 2
+  br i1 %cc3, label %bb.inner.then2, label %bb.outer.end
+
+bb.inner.then2:
+  %tmp5 = getelementptr inbounds i32, i32 addrspace(1)* %tmp1, i32 4
+  store i32 4, i32 addrspace(1)* %tmp5, align 4
+  br label %bb.outer.end
+
+bb.outer.end:
+  ret void
+}
+
+; GCN-LABEL: {{^}}s_endpgm_unsafe_barrier:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]]
+; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9_]+]]
+; GCN-NEXT: {{^BB[0-9_]+}}:
+; GCN:      store_dword
+; GCN-NEXT: {{^}}[[ENDIF]]:
+; GCN-NEXT: s_or_b64 exec, exec, [[SAVEEXEC]]
+; GCN:      s_barrier
+; GCN-NEXT: s_endpgm
+define amdgpu_kernel void @s_endpgm_unsafe_barrier(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = icmp ugt i32 %tmp, 1
+  br i1 %tmp1, label %bb.then, label %bb.end
+
+bb.then:                                          ; preds = %bb
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp
+  store i32 0, i32 addrspace(1)* %tmp4, align 4
+  br label %bb.end
+
+bb.end:                                           ; preds = %bb.then, %bb
+  call void @llvm.amdgcn.s.barrier()
+  ret void
+}
+
+; Make sure scc liveness is updated if sor_b64 is removed
+; GCN-LABEL: {{^}}scc_liveness:
+
+; GCN: [[BB1_LOOP:BB[0-9]+_[0-9]+]]:
+; GCN: s_andn2_b64 exec, exec,
+; GCN-NEXT: s_cbranch_execnz [[BB1_LOOP]]
+
+; GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen
+; GCN: s_and_b64 exec, exec, vcc
+
+; GCN-NOT: s_or_b64 exec, exec
+
+; GCN: s_or_b64 exec, exec, s{{\[[0-9]+:[0-9]+\]}}
+; GCN: s_andn2_b64
+; GCN-NEXT: s_cbranch_execnz
+
+; GCN: s_or_b64 exec, exec, s{{\[[0-9]+:[0-9]+\]}}
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+; GCN: buffer_store_dword
+; GCN: s_setpc_b64
+define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %Flow1, %bb1, %bb
+  %tmp = icmp slt i32 %arg, 519
+  br i1 %tmp, label %bb2, label %bb1
+
+bb2:                                              ; preds = %bb1
+  %tmp3 = icmp eq i32 %arg, 0
+  br i1 %tmp3, label %bb4, label %bb10
+
+bb4:                                              ; preds = %bb2
+  %tmp6 = load float, float* undef
+  %tmp7 = fcmp olt float %tmp6, 0.0
+  br i1 %tmp7, label %bb8, label %Flow
+
+bb8:                                              ; preds = %bb4
+  %tmp9 = insertelement <4 x float> undef, float 0.0, i32 1
+  br label %Flow
+
+Flow:                                             ; preds = %bb8, %bb4
+  %tmp8 = phi <4 x float> [ %tmp9, %bb8 ], [ zeroinitializer, %bb4 ]
+  br label %bb10
+
+bb10:                                             ; preds = %Flow, %bb2
+  %tmp11 = phi <4 x float> [ zeroinitializer, %bb2 ], [ %tmp8, %Flow ]
+  br i1 %tmp3, label %bb12, label %Flow1
+
+Flow1:                                            ; preds = %bb10
+  br label %bb1
+
+bb12:                                             ; preds = %bb10
+  store volatile <4 x float> %tmp11, <4 x float>* undef, align 16
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+declare void @llvm.amdgcn.s.barrier() #1
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind convergent }
+attributes #2 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/combine-ftrunc.ll b/test/CodeGen/AMDGPU/combine-ftrunc.ll
new file mode 100644
index 0000000000000..77ab7c3a948c3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/combine-ftrunc.ll
@@ -0,0 +1,92 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}combine_ftrunc_frint_f64:
+; GCN: v_rndne_f64_e32 [[RND:v\[[0-9:]+\]]],
+; GCN: flat_store_dwordx2 v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_frint_f64(double addrspace(1)* %p) {
+  %v = load double, double addrspace(1)* %p, align 8
+  %round = tail call double @llvm.rint.f64(double %v)
+  %trunc = tail call double @llvm.trunc.f64(double %round)
+  store double %trunc, double addrspace(1)* %p, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_frint_f32:
+; GCN: v_rndne_f32_e32 [[RND:v[0-9]+]],
+; GCN: flat_store_dword v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_frint_f32(float addrspace(1)* %p) {
+  %v = load float, float addrspace(1)* %p, align 4
+  %round = tail call float @llvm.rint.f32(float %v)
+  %trunc = tail call float @llvm.trunc.f32(float %round)
+  store float %trunc, float addrspace(1)* %p, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_frint_v2f32:
+; GCN: s_load_dwordx2
+; GCN: s_load_dwordx2 s{{\[}}[[SRC1:[0-9]+]]:[[SRC2:[0-9]+]]{{\]}}
+; GCN-DAG: v_rndne_f32_e32 v[[RND1:[0-9]+]], s[[SRC1]]
+; GCN-DAG: v_rndne_f32_e32 v[[RND2:[0-9]+]], s[[SRC2]]
+; GCN: flat_store_dwordx2 v[{{[0-9:]+}}], v{{\[}}[[RND1]]:[[RND2]]{{\]}}
+define amdgpu_kernel void @combine_ftrunc_frint_v2f32(<2 x float> addrspace(1)* %p) {
+  %v = load <2 x float>, <2 x float> addrspace(1)* %p, align 8
+  %round = tail call <2 x float> @llvm.rint.v2f32(<2 x float> %v)
+  %trunc = tail call <2 x float> @llvm.trunc.v2f32(<2 x float> %round)
+  store <2 x float> %trunc, <2 x float> addrspace(1)* %p, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_fceil_f32:
+; GCN: v_ceil_f32_e32 [[RND:v[0-9]+]],
+; GCN: flat_store_dword v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_fceil_f32(float addrspace(1)* %p) {
+  %v = load float, float addrspace(1)* %p, align 4
+  %round = tail call float @llvm.ceil.f32(float %v)
+  %trunc = tail call float @llvm.trunc.f32(float %round)
+  store float %trunc, float addrspace(1)* %p, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_ffloor_f32:
+; GCN: v_floor_f32_e32 [[RND:v[0-9]+]],
+; GCN: flat_store_dword v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_ffloor_f32(float addrspace(1)* %p) {
+  %v = load float, float addrspace(1)* %p, align 4
+  %round = tail call float @llvm.floor.f32(float %v)
+  %trunc = tail call float @llvm.trunc.f32(float %round)
+  store float %trunc, float addrspace(1)* %p, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_fnearbyint_f32:
+; GCN: v_rndne_f32_e32 [[RND:v[0-9]+]],
+; GCN: flat_store_dword v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_fnearbyint_f32(float addrspace(1)* %p) {
+  %v = load float, float addrspace(1)* %p, align 4
+  %round = tail call float @llvm.nearbyint.f32(float %v)
+  %trunc = tail call float @llvm.trunc.f32(float %round)
+  store float %trunc, float addrspace(1)* %p, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}combine_ftrunc_ftrunc_f32:
+; GCN: s_load_dword [[SRC:s[0-9]+]],
+; GCN: v_trunc_f32_e32 [[RND:v[0-9]+]], [[SRC]]
+; GCN: flat_store_dword v[{{[0-9:]+}}], [[RND]]
+define amdgpu_kernel void @combine_ftrunc_ftrunc_f32(float addrspace(1)* %p) {
+  %v = load float, float addrspace(1)* %p, align 4
+  %round = tail call float @llvm.trunc.f32(float %v)
+  %trunc = tail call float @llvm.trunc.f32(float %round)
+  store float %trunc, float addrspace(1)* %p, align 4
+  ret void
+}
+
+declare double @llvm.trunc.f64(double)
+declare float @llvm.trunc.f32(float)
+declare <2 x float> @llvm.trunc.v2f32(<2 x float>)
+declare double @llvm.rint.f64(double)
+declare float @llvm.rint.f32(float)
+declare <2 x float> @llvm.rint.v2f32(<2 x float>)
+declare float @llvm.ceil.f32(float)
+declare float @llvm.floor.f32(float)
+declare float @llvm.nearbyint.f32(float)
diff --git a/test/CodeGen/AMDGPU/combine_vloads.ll b/test/CodeGen/AMDGPU/combine_vloads.ll
index f8d4e01085c26..005cfe6f955eb 100644
--- a/test/CodeGen/AMDGPU/combine_vloads.ll
+++ b/test/CodeGen/AMDGPU/combine_vloads.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
 
 ;
-; kernel void combine_vloads(global char8* src, global char8* result) {
+; kernel void combine_vloads(global char8 addrspace(5)* src, global char8 addrspace(5)* result) {
 ;   for (int i = 0; i < 1024; ++i)
 ;     result[i] = src[0] + src[1] + src[2] + src[3];
 ; }
diff --git a/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir b/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
index 0401f7b07e218..d29c6afe7d4d6 100644
--- a/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
+++ b/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
@@ -2,7 +2,7 @@
 ...
 
 # GCN-LABEL: name: s_fold_and_imm_regimm_32{{$}}
-# GCN: %10 = V_MOV_B32_e32 1543, implicit %exec
+# GCN: %10:vgpr_32 = V_MOV_B32_e32 1543, implicit %exec
 # GCN: BUFFER_STORE_DWORD_OFFSET killed %10,
 name:            s_fold_and_imm_regimm_32
 alignment:       0
@@ -62,19 +62,19 @@ body:             |
 
 # GCN-LABEL: name: v_fold_and_imm_regimm_32{{$}}
 
-# GCN: %9 = V_MOV_B32_e32 646, implicit %exec
+# GCN: %9:vgpr_32 = V_MOV_B32_e32 646, implicit %exec
 # GCN: FLAT_STORE_DWORD %19, %9,
 
-# GCN: %10 = V_MOV_B32_e32 646, implicit %exec
+# GCN: %10:vgpr_32 = V_MOV_B32_e32 646, implicit %exec
 # GCN: FLAT_STORE_DWORD %19, %10
 
-# GCN: %11 = V_MOV_B32_e32 646, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 646, implicit %exec
 # GCN: FLAT_STORE_DWORD %19, %11,
 
-# GCN: %12 = V_MOV_B32_e32 1234567, implicit %exec
+# GCN: %12:vgpr_32 = V_MOV_B32_e32 1234567, implicit %exec
 # GCN: FLAT_STORE_DWORD %19, %12,
 
-# GCN: %13 = V_MOV_B32_e32 63, implicit %exec
+# GCN: %13:vgpr_32 = V_MOV_B32_e32 63, implicit %exec
 # GCN: FLAT_STORE_DWORD %19, %13,
 
 name:            v_fold_and_imm_regimm_32
@@ -226,34 +226,34 @@ body:             |
 ---
 # GCN-LABEL: name: v_fold_shl_imm_regimm_32{{$}}
 
-# GCN: %11 = V_MOV_B32_e32 40955904, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 40955904, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %11,
 
-# GCN: %12 = V_MOV_B32_e32 24, implicit %exec
+# GCN: %12:vgpr_32 = V_MOV_B32_e32 24, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %12,
 
-# GCN: %13 = V_MOV_B32_e32 4096, implicit %exec
+# GCN: %13:vgpr_32 = V_MOV_B32_e32 4096, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %13,
 
-# GCN: %14 = V_MOV_B32_e32 24, implicit %exec
+# GCN: %14:vgpr_32 = V_MOV_B32_e32 24, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %14,
 
-# GCN: %15 = V_MOV_B32_e32 0, implicit %exec
+# GCN: %15:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %15,
 
-# GCN: %22 = V_MOV_B32_e32 4096, implicit %exec
+# GCN: %22:vgpr_32 = V_MOV_B32_e32 4096, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %22,
 
-# GCN: %23 = V_MOV_B32_e32 1, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 1, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %23,
 
-# GCN: %25 = V_MOV_B32_e32 2, implicit %exec
+# GCN: %25:vgpr_32 = V_MOV_B32_e32 2, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %25,
 
-# GCN: %26 = V_MOV_B32_e32 7927808, implicit %exec
+# GCN: %26:vgpr_32 = V_MOV_B32_e32 7927808, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %26,
 
-# GCN: %28 = V_MOV_B32_e32 -8, implicit %exec
+# GCN: %28:vgpr_32 = V_MOV_B32_e32 -8, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %28,
 
 name:            v_fold_shl_imm_regimm_32
@@ -367,7 +367,7 @@ body:             |
 ---
 
 # GCN-LABEL: name: s_fold_ashr_imm_regimm_32{{$}}
-# GCN: %11 = V_MOV_B32_e32 243, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 243, implicit %exec
 # GCN: BUFFER_STORE_DWORD_OFFSET killed %11, killed %8,
 name:            s_fold_ashr_imm_regimm_32
 alignment:       0
@@ -425,34 +425,34 @@ body:             |
 ...
 
 # GCN-LABEL: name: v_fold_ashr_imm_regimm_32{{$}}
-# GCN: %11 = V_MOV_B32_e32 3903258, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 3903258, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %11,
 
-# GCN: %12 = V_MOV_B32_e32 62452139, implicit %exec
+# GCN: %12:vgpr_32 = V_MOV_B32_e32 62452139, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %12,
 
-# GCN: %13 = V_MOV_B32_e32 1678031, implicit %exec
+# GCN: %13:vgpr_32 = V_MOV_B32_e32 1678031, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %13,
 
-# GCN: %14 = V_MOV_B32_e32 3, implicit %exec
+# GCN: %14:vgpr_32 = V_MOV_B32_e32 3, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %14,
 
-# GCN: %15 = V_MOV_B32_e32 -1, implicit %exec
+# GCN: %15:vgpr_32 = V_MOV_B32_e32 -1, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %15,
 
-# GCN: %22 = V_MOV_B32_e32 62500, implicit %exec
+# GCN: %22:vgpr_32 = V_MOV_B32_e32 62500, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %22,
 
-# GCN: %23 = V_MOV_B32_e32 500000, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 500000, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %23,
 
-# GCN: %25 = V_MOV_B32_e32 1920, implicit %exec
+# GCN: %25:vgpr_32 = V_MOV_B32_e32 1920, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %25,
 
-# GCN: %26 = V_MOV_B32_e32 487907, implicit %exec
+# GCN: %26:vgpr_32 = V_MOV_B32_e32 487907, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %26,
 
-# GCN: %28 = V_MOV_B32_e32 -1, implicit %exec
+# GCN: %28:vgpr_32 = V_MOV_B32_e32 -1, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %28,
 
 name:            v_fold_ashr_imm_regimm_32
@@ -575,7 +575,7 @@ body:             |
 ---
 
 # GCN-LABEL: name: s_fold_lshr_imm_regimm_32{{$}}
-# GCN: %11 = V_MOV_B32_e32 1048332, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 1048332, implicit %exec
 # GCN: BUFFER_STORE_DWORD_OFFSET killed %11, killed %8,
 name:            s_fold_lshr_imm_regimm_32
 alignment:       0
@@ -634,34 +634,34 @@ body:             |
 ---
 
 # GCN-LABEL: name: v_fold_lshr_imm_regimm_32{{$}}
-# GCN: %11 = V_MOV_B32_e32 3903258, implicit %exec
+# GCN: %11:vgpr_32 = V_MOV_B32_e32 3903258, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %11,
 
-# GCN: %12 = V_MOV_B32_e32 62452139, implicit %exec
+# GCN: %12:vgpr_32 = V_MOV_B32_e32 62452139, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %12,
 
-# GCN: %13 = V_MOV_B32_e32 1678031, implicit %exec
+# GCN: %13:vgpr_32 = V_MOV_B32_e32 1678031, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %13,
 
-# GCN: %14 = V_MOV_B32_e32 3, implicit %exec
+# GCN: %14:vgpr_32 = V_MOV_B32_e32 3, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %14,
 
-# GCN: %15 = V_MOV_B32_e32 1, implicit %exec
+# GCN: %15:vgpr_32 = V_MOV_B32_e32 1, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %15,
 
-# GCN: %22 = V_MOV_B32_e32 62500, implicit %exec
+# GCN: %22:vgpr_32 = V_MOV_B32_e32 62500, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %22,
 
-# GCN: %23 = V_MOV_B32_e32 500000, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 500000, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %23,
 
-# GCN: %25 = V_MOV_B32_e32 1920, implicit %exec
+# GCN: %25:vgpr_32 = V_MOV_B32_e32 1920, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %25,
 
-# GCN: %26 = V_MOV_B32_e32 487907, implicit %exec
+# GCN: %26:vgpr_32 = V_MOV_B32_e32 487907, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %26,
 
-# GCN: %28 = V_MOV_B32_e32 1073741823, implicit %exec
+# GCN: %28:vgpr_32 = V_MOV_B32_e32 1073741823, implicit %exec
 # GCN: FLAT_STORE_DWORD %20, %28,
 
 name:            v_fold_lshr_imm_regimm_32
@@ -787,7 +787,7 @@ body:             |
 
 # GCN-LABEL: name: undefined_vreg_operand{{$}}
 # GCN: bb.0
-# GCN-NEXT: FLAT_STORE_DWORD undef %3, undef %1,
+# GCN-NEXT: FLAT_STORE_DWORD undef %3:vreg_64, undef %1:vgpr_32,
 # GCN-NEXT: S_ENDPGM
 name: undefined_vreg_operand
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
index 79d9b16918783..071bcbcf81bfd 100644
--- a/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
+++ b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -check-prefix=VMEM -check-prefix=GCN %s
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=1 -verify-machineinstrs < %s | FileCheck -check-prefix=VGPR -check-prefix=GCN %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VMEM -check-prefix=GCN %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=1 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=VGPR -check-prefix=GCN %s
 
 ; Verify registers used for tracking exec mask changes when all
 ; registers are spilled at the end of the block. The SGPR spill
@@ -13,14 +13,13 @@
 ; VGPR: workitem_private_segment_byte_size = 12{{$}}
 
 
-; GCN: {{^}}; BB#0:
+; GCN: {{^}}; %bb.0:
 ; GCN: s_mov_b32 m0, -1
 ; GCN: ds_read_b32 [[LOAD0:v[0-9]+]]
 
 ; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0
 ; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec
 ; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, [[CMP0]]
-; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}
 
 ; Spill saved exec
 ; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]]
@@ -90,9 +89,9 @@ endif:
 }
 
 ; GCN-LABEL: {{^}}divergent_loop:
-; VGPR: workitem_private_segment_byte_size = 16{{$}}
+; VGPR: workitem_private_segment_byte_size = 12{{$}}
 
-; GCN: {{^}}; BB#0:
+; GCN: {{^}}; %bb.0:
 
 ; GCN: s_mov_b32 m0, -1
 ; GCN: ds_read_b32 [[LOAD0:v[0-9]+]]
@@ -101,7 +100,6 @@ endif:
 
 ; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec
 ; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]]
-; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}
 
 ; Spill load
 ; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 offset:4 ; 4-byte Folded Spill
@@ -125,10 +123,9 @@ endif:
 ; GCN: [[LOOP:BB[0-9]+_[0-9]+]]:
 ; GCN: buffer_load_dword v[[VAL_LOOP_RELOAD:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload
 ; GCN: v_subrev_i32_e32 [[VAL_LOOP:v[0-9]+]], vcc, v{{[0-9]+}}, v[[VAL_LOOP_RELOAD]]
-; GCN: v_cmp_ne_u32_e32 vcc,
-; GCN: s_and_b64 vcc, exec, vcc
+; GCN: s_cmp_lg_u32 s{{[0-9]+}}, s{{[0-9]+}}
 ; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill
-; GCN-NEXT: s_cbranch_vccnz [[LOOP]]
+; GCN-NEXT: s_cbranch_scc1 [[LOOP]]
 
 
 ; GCN: [[END]]:
@@ -170,10 +167,10 @@ end:
 }
 
 ; GCN-LABEL: {{^}}divergent_if_else_endif:
-; GCN: {{^}}; BB#0:
+; GCN: {{^}}; %bb.0:
 
 ; GCN: s_mov_b32 m0, -1
-; VMEM: ds_read_b32 [[LOAD0:v[0-9]+]]
+; GCN: ds_read_b32 [[LOAD0:v[0-9]+]]
 
 ; GCN: v_cmp_ne_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], v0,
 
diff --git a/test/CodeGen/AMDGPU/control-flow-optnone.ll b/test/CodeGen/AMDGPU/control-flow-optnone.ll
new file mode 100644
index 0000000000000..2122af62735fe
--- /dev/null
+++ b/test/CodeGen/AMDGPU/control-flow-optnone.ll
@@ -0,0 +1,54 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; optnone disables AMDGPUAnnotateUniformValues, so no branch is known
+; to be uniform during instruction selection. The custom selection for
+; brcond was not checking if the branch was uniform, relying on the
+; selection pattern to check that. That would fail, so then the branch
+; would fail to select.
+
+; GCN-LABEL: {{^}}copytoreg_divergent_brcond:
+; GCN: s_branch
+
+; GCN-DAG: v_cmp_lt_i32
+; GCN-DAG: v_cmp_gt_i32
+; GCN: s_and_b64
+; GCN: s_mov_b64 exec
+
+; GCN: s_or_b64 exec, exec
+; GCN: v_cmp_eq_u32
+; GCN: s_cbranch_vccnz
+; GCN-NEXT: s_branch
+define amdgpu_kernel void @copytoreg_divergent_brcond(i32 %arg, i32 %arg1, i32 %arg2) #0 {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp3 = zext i32 %tmp to i64
+  %tmp5 = add i64 %tmp3, undef
+  %tmp6 = trunc i64 %tmp5 to i32
+  %tmp7 = mul nsw i32 %tmp6, %arg2
+  br label %bb8
+
+bb8.loopexit:                                     ; preds = %bb14
+  br label %bb8
+
+bb8:                                              ; preds = %bb8.loopexit, %bb
+  br label %bb9
+
+bb9:                                              ; preds = %bb14, %bb8
+  %tmp10 = icmp slt i32 %tmp7, %arg1
+  %tmp11 = icmp sgt i32 %arg, 0
+  %tmp12 = and i1 %tmp10, %tmp11
+  br i1 %tmp12, label %bb13, label %bb14
+
+bb13:                                             ; preds = %bb9
+  store volatile i32 0, i32 addrspace(1)* undef, align 4
+  br label %bb14
+
+bb14:                                             ; preds = %bb13, %bb9
+  %tmp15 = icmp eq i32 %arg2, 1
+  br i1 %tmp15, label %bb8.loopexit, label %bb9
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind optnone noinline }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/convergent-inlineasm.ll b/test/CodeGen/AMDGPU/convergent-inlineasm.ll
index 0074a41e44cf8..80907bf1c1beb 100644
--- a/test/CodeGen/AMDGPU/convergent-inlineasm.ll
+++ b/test/CodeGen/AMDGPU/convergent-inlineasm.ll
@@ -2,7 +2,7 @@
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 ; GCN-LABEL: {{^}}convergent_inlineasm:
-; GCN: BB#0:
+; GCN: %bb.0:
 ; GCN: v_cmp_ne_u32_e64
 ; GCN: ; mask branch
 ; GCN: BB{{[0-9]+_[0-9]+}}:
diff --git a/test/CodeGen/AMDGPU/ctlz.ll b/test/CodeGen/AMDGPU/ctlz.ll
index a544cbe890b50..40548ba4cb9b1 100644
--- a/test/CodeGen/AMDGPU/ctlz.ll
+++ b/test/CodeGen/AMDGPU/ctlz.ll
@@ -112,7 +112,7 @@ define amdgpu_kernel void @v_ctlz_v4i32(<4 x i32> addrspace(1)* noalias %out, <4
 ; GCN: v_cndmask_b32_e32 [[SELECT:v[0-9]+]], 32, [[FFBH]], vcc
 
 ; SI: v_subrev_i32_e32 [[RESULT:v[0-9]+]], vcc, 24, [[SELECT]]
-; VI: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, -16, [[SELECT]]
+; VI: v_add_u32_e32 [[RESULT:v[0-9]+]], vcc, -16, [[SELECT]]
 ; GCN: buffer_store_byte [[RESULT]],
 ; GCN: s_endpgm
 define amdgpu_kernel void @v_ctlz_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind {
@@ -151,7 +151,7 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64
 ; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
 ; GCN-DAG: v_cmp_eq_u32_e32 vcc, 0, v[[HI]]
 ; GCN-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]]
-; GCN-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
+; GCN-DAG: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
 ; GCN-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]]
 ; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[ADD]], vcc
 ; GCN-DAG: v_or_b32_e32 [[OR:v[0-9]+]], v[[LO]], v[[HI]]
diff --git a/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index 7500da536307f..9796bc780edc4 100644
--- a/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -126,7 +126,7 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(i32 addrspace(1)* noalias
 ; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
 ; GCN-DAG: v_cmp_eq_u32_e32 vcc, 0, v[[HI]]
 ; GCN-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]]
-; GCN-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
+; GCN-DAG: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]]
 ; GCN-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]]
 ; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[FFBH_LO]]
 ; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI:[0-9]+]]{{\]}}
diff --git a/test/CodeGen/AMDGPU/ctpop.ll b/test/CodeGen/AMDGPU/ctpop.ll
index 68b39bad2bc12..2d16d4034d36c 100644
--- a/test/CodeGen/AMDGPU/ctpop.ll
+++ b/test/CodeGen/AMDGPU/ctpop.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC -check-prefix=VI %s
-; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=FUNC -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=FUNC -check-prefix=VI %s
+; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
 
 declare i32 @llvm.ctpop.i32(i32) nounwind readnone
 declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone
@@ -42,8 +42,10 @@ define amdgpu_kernel void @v_ctpop_i32(i32 addrspace(1)* noalias %out, i32 addrs
 }
 
 ; FUNC-LABEL: {{^}}v_ctpop_add_chain_i32:
-; GCN: {{buffer|flat}}_load_dword [[VAL0:v[0-9]+]],
-; GCN: {{buffer|flat}}_load_dword [[VAL1:v[0-9]+]],
+; SI: buffer_load_dword [[VAL0:v[0-9]+]],
+; SI: buffer_load_dword [[VAL1:v[0-9]+]],
+; VI: flat_load_dword [[VAL0:v[0-9]+]],
+; VI: flat_load_dword [[VAL1:v[0-9]+]],
 ; GCN: v_bcnt_u32_b32{{(_e64)*}} [[MIDRESULT:v[0-9]+]], [[VAL1]], 0
 ; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]]
 ; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]]
@@ -56,8 +58,8 @@ define amdgpu_kernel void @v_ctpop_add_chain_i32(i32 addrspace(1)* noalias %out,
   %tid = call i32 @llvm.r600.read.tidig.x()
   %in0.gep = getelementptr i32, i32 addrspace(1)* %in0, i32 %tid
   %in1.gep = getelementptr i32, i32 addrspace(1)* %in1, i32 %tid
-  %val0 = load i32, i32 addrspace(1)* %in0.gep, align 4
-  %val1 = load i32, i32 addrspace(1)* %in1.gep, align 4
+  %val0 = load volatile i32, i32 addrspace(1)* %in0.gep, align 4
+  %val1 = load volatile i32, i32 addrspace(1)* %in1.gep, align 4
   %ctpop0 = call i32 @llvm.ctpop.i32(i32 %val0) nounwind readnone
   %ctpop1 = call i32 @llvm.ctpop.i32(i32 %val1) nounwind readnone
   %add = add i32 %ctpop0, %ctpop1
@@ -280,8 +282,8 @@ define amdgpu_kernel void @v_ctpop_i32_add_var_inv(i32 addrspace(1)* noalias %ou
 ; SI: buffer_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64
 ; SI: buffer_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64
 ; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]
-; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
 ; VI: flat_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
+; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]
 ; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
@@ -306,7 +308,7 @@ define amdgpu_kernel void @v_ctpop_i32_add_vvar_inv(i32 addrspace(1)* noalias %o
 ; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xd
 ; VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x34
 ; GCN: s_bcnt1_i32_b32  [[SRESULT:s[0-9]+]], [[VAL]]
-; GCN: v_mov_b32_e32 [[RESULT]], [[SRESULT]]
+; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
 ; EG: BCNT_INT
diff --git a/test/CodeGen/AMDGPU/ctpop64.ll b/test/CodeGen/AMDGPU/ctpop64.ll
index 4850370851f63..298280f755ff8 100644
--- a/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/test/CodeGen/AMDGPU/ctpop64.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -188,7 +188,7 @@ define amdgpu_kernel void @s_ctpop_i65(i32 addrspace(1)* noalias %out, i65 %val)
 ; GCN-DAG: v_bcnt_u32_b32{{(_e64)*}} [[MIDRESULT2:v[0-9]+]], v[[VAL0]], 0
 ; GCN-DAG: v_bcnt_u32_b32{{(_e32)*(_e64)*}} [[MIDRESULT3:v[0-9]+]], v{{[0-9]+}}, [[MIDRESULT2]]
 
-; GCN: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, [[MIDRESULT1]], [[MIDRESULT2]]
+; GCN: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, [[MIDRESULT1]], [[MIDRESULT2]]
 
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
diff --git a/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 1bfd38d94bfdf..71b5fd9ba7ae6 100644
--- a/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -1,8 +1,12 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-NOSDWA -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-SDWA  -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
 
+declare i7 @llvm.cttz.i7(i7, i1) nounwind readnone
+declare i8 @llvm.cttz.i8(i8, i1) nounwind readnone
+declare i16 @llvm.cttz.i16(i16, i1) nounwind readnone
 declare i32 @llvm.cttz.i32(i32, i1) nounwind readnone
+declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
 declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) nounwind readnone
 declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1) nounwind readnone
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
@@ -76,3 +80,190 @@ define amdgpu_kernel void @v_cttz_zero_undef_v4i32(<4 x i32> addrspace(1)* noali
   store <4 x i32> %cttz, <4 x i32> addrspace(1)* %out, align 16
   ret void
 }
+
+; FUNC-LABEL: {{^}}s_cttz_zero_undef_i8_with_select:
+; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}}
+; EG: MEM_RAT MSKOR
+; EG: FFBL_INT
+define amdgpu_kernel void @s_cttz_zero_undef_i8_with_select(i8 addrspace(1)* noalias %out, i8 %val) nounwind {
+  %cttz = tail call i8 @llvm.cttz.i8(i8 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i8 %val, 0
+  %ret = select i1 %cttz_ret, i8 %cttz, i8 32
+  store i8 %cttz, i8 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_cttz_zero_undef_i16_with_select:
+; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}}
+; EG: MEM_RAT MSKOR
+; EG: FFBL_INT
+define amdgpu_kernel void @s_cttz_zero_undef_i16_with_select(i16 addrspace(1)* noalias %out, i16 %val) nounwind {
+  %cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i16 %val, 0
+  %ret = select i1 %cttz_ret, i16 %cttz, i16 32
+  store i16 %cttz, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_cttz_zero_undef_i32_with_select:
+; SI: s_ff1_i32_b32
+; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
+; EG: FFBL_INT {{\*? *}}[[RESULT]]
+define amdgpu_kernel void @s_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 %val) nounwind {
+  %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i32 %val, 0
+  %ret = select i1 %cttz_ret, i32 %cttz, i32 32
+  store i32 %cttz, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_cttz_zero_undef_i64_with_select:
+; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}}
+; SI: s_ff1_i32_b32 s{{[0-9]+}}, s{{[0-9]+}}
+; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
+define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(i64 addrspace(1)* noalias %out, i64 %val) nounwind {
+  %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i64 %val, 0
+  %ret = select i1 %cttz_ret, i64 %cttz, i64 32
+  store i64 %cttz, i64 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_zero_undef_i8_with_select:
+; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI-SDWA: v_ffbl_b32_sdwa
+; EG: MEM_RAT MSKOR
+define amdgpu_kernel void @v_cttz_zero_undef_i8_with_select(i8 addrspace(1)* noalias %out, i8 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i8, i8 addrspace(1)* %arrayidx, align 1
+  %cttz = tail call i8 @llvm.cttz.i8(i8 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i8 %val, 0
+  %ret = select i1 %cttz_ret, i8 %cttz, i8 32
+  store i8 %ret, i8 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_zero_undef_i16_with_select:
+; SI-NOSDWA: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI-SDWA: v_ffbl_b32_sdwa
+; EG: MEM_RAT MSKOR
+define amdgpu_kernel void @v_cttz_zero_undef_i16_with_select(i16 addrspace(1)* noalias %out, i16 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i16, i16 addrspace(1)* %arrayidx, align 1
+  %cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i16 %val, 0
+  %ret = select i1 %cttz_ret, i16 %cttz, i16 32
+  store i16 %ret, i16 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_zero_undef_i32_with_select:
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_cmp_ne_u32_e32 vcc, 0
+; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
+define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i32, i32 addrspace(1)* %arrayidx, align 1
+  %cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i32 %val, 0
+  %ret = select i1 %cttz_ret, i32 %cttz, i32 32
+  store i32 %ret, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_zero_undef_i64_with_select:
+; SI-NOSDWA: v_or_b32_e32
+; SI-NOSDWA: v_or_b32_e32
+; SI-NOSDWA: v_or_b32_e32
+; SI-SDWA: v_or_b32_sdwa
+; SI-NOSDWA: v_or_b32_e32
+; SI-SDWA: v_or_b32_sdwa
+; SI: v_or_b32_e32 [[VAL1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_or_b32_e32 [[VAL2:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
+; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL1]]
+; SI-DAG: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL2]]
+; SI: v_cmp_eq_u32_e32 vcc, 0
+; SI: v_cmp_ne_u64_e32 vcc, 0
+; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]]
+define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(i64 addrspace(1)* noalias %out, i64 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i64, i64 addrspace(1)* %arrayidx, align 1
+  %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
+  %cttz_ret = icmp ne i64 %val, 0
+  %ret = select i1 %cttz_ret, i64 %cttz, i64 32
+  store i64 %ret, i64 addrspace(1)* %out, align 4
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_i32_sel_eq_neg1:
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL:v[0-9]+]]
+; SI: v_cmp_ne_u32_e32 vcc, 0, [[VAL]]
+; SI: s_endpgm
+; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG: FFBL_INT
+define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i32, i32 addrspace(1)* %arrayidx, align 1
+  %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
+  %cmp = icmp eq i32 %val, 0
+  %sel = select i1 %cmp, i32 -1, i32 %ctlz
+  store i32 %sel, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_i32_sel_ne_neg1:
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, [[VAL:v[0-9]+]]
+; SI: v_cmp_ne_u32_e32 vcc, 0, [[VAL]]
+; SI: s_endpgm
+; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG: FFBL_INT
+define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i32, i32 addrspace(1)* %arrayidx, align 1
+  %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
+  %cmp = icmp ne i32 %val, 0
+  %sel = select i1 %cmp, i32 %ctlz, i32 -1
+  store i32 %sel, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_i32_sel_ne_bitwidth:
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_cmp
+; SI: v_cndmask
+; SI: s_endpgm
+; EG: MEM_RAT_CACHELESS STORE_RAW
+; EG: FFBL_INT
+define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i32, i32 addrspace(1)* %arrayidx, align 1
+  %ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
+  %cmp = icmp ne i32 %ctlz, 32
+  %sel = select i1 %cmp, i32 %ctlz, i32 -1
+  store i32 %sel, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_i8_sel_eq_neg1:
+; SI: {{buffer|flat}}_load_ubyte
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; EG: MEM_RAT MSKOR
+; EG: FFBL_INT
+ define amdgpu_kernel void @v_cttz_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i8, i8 addrspace(1)* %arrayidx, align 1
+  %ctlz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
+  %cmp = icmp eq i8 %val, 0
+  %sel = select i1 %cmp, i8 -1, i8 %ctlz
+  store i8 %sel, i8 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}v_cttz_i16_sel_eq_neg1:
+; SI: {{buffer|flat}}_load_ubyte
+; SI: v_ffbl_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: buffer_store_short
+; EG: MEM_RAT MSKOR
+; EG: FFBL_INT
+ define amdgpu_kernel void @v_cttz_i16_sel_eq_neg1(i16 addrspace(1)* noalias %out, i16 addrspace(1)* nocapture readonly %arrayidx) nounwind {
+  %val = load i16, i16 addrspace(1)* %arrayidx, align 1
+  %ctlz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone
+  %cmp = icmp eq i16 %val, 0
+  %sel = select i1 %cmp, i16 -1, i16 %ctlz
+  store i16 %sel, i16 addrspace(1)* %out
+  ret void
+}
+
+
diff --git a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index f839129fc3d87..c6f7e3ea19a40 100644
--- a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -12,7 +12,7 @@ declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
 ; GCN: buffer_store_dword [[CONV]],
 define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i8, i8 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i8, i8 addrspace(1)* %in, i32 %tid
   %load = load i8, i8 addrspace(1)* %gep, align 1
   %cvt = uitofp i8 %load to float
   store float %cvt, float addrspace(1)* %out, align 4
@@ -26,7 +26,7 @@ define amdgpu_kernel void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 a
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}},
 define amdgpu_kernel void @load_v2i8_to_v2f32(<2 x float> addrspace(1)* noalias %out, <2 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <2 x i8>, <2 x i8> addrspace(1)* %in, i32 %tid
   %load = load <2 x i8>, <2 x i8> addrspace(1)* %gep, align 2
   %cvt = uitofp <2 x i8> %load to <2 x float>
   store <2 x float> %cvt, <2 x float> addrspace(1)* %out, align 16
@@ -42,7 +42,7 @@ define amdgpu_kernel void @load_v2i8_to_v2f32(<2 x float> addrspace(1)* noalias
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}},
 define amdgpu_kernel void @load_v3i8_to_v3f32(<3 x float> addrspace(1)* noalias %out, <3 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %in, i32 %tid
   %load = load <3 x i8>, <3 x i8> addrspace(1)* %gep, align 4
   %cvt = uitofp <3 x i8> %load to <3 x float>
   store <3 x float> %cvt, <3 x float> addrspace(1)* %out, align 16
@@ -60,7 +60,7 @@ define amdgpu_kernel void @load_v3i8_to_v3f32(<3 x float> addrspace(1)* noalias
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}},
 define amdgpu_kernel void @load_v4i8_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 4
   %cvt = uitofp <4 x i8> %load to <4 x float>
   store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16
@@ -86,7 +86,7 @@ define amdgpu_kernel void @load_v4i8_to_v4f32(<4 x float> addrspace(1)* noalias
 ; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 1
   %cvt = uitofp <4 x i8> %load to <4 x float>
   store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16
@@ -135,7 +135,7 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* n
 ; GCN: s_endpgm
 define amdgpu_kernel void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias %out, <7 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <7 x i8>, <7 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <7 x i8>, <7 x i8> addrspace(1)* %in, i32 %tid
   %load = load <7 x i8>, <7 x i8> addrspace(1)* %gep, align 1
   %cvt = uitofp <7 x i8> %load to <7 x float>
   store <7 x float> %cvt, <7 x float> addrspace(1)* %out, align 16
@@ -160,7 +160,7 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias
 ; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @load_v8i8_to_v8f32(<8 x float> addrspace(1)* noalias %out, <8 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <8 x i8>, <8 x i8> addrspace(1)* %in, i32 %tid
   %load = load <8 x i8>, <8 x i8> addrspace(1)* %gep, align 8
   %cvt = uitofp <8 x i8> %load to <8 x float>
   store <8 x float> %cvt, <8 x float> addrspace(1)* %out, align 16
@@ -169,12 +169,12 @@ define amdgpu_kernel void @load_v8i8_to_v8f32(<8 x float> addrspace(1)* noalias
 
 ; GCN-LABEL: {{^}}i8_zext_inreg_i32_to_f32:
 ; GCN: {{buffer|flat}}_load_dword [[LOADREG:v[0-9]+]],
-; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 2, [[LOADREG]]
+; GCN: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 2, [[LOADREG]]
 ; GCN-NEXT: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[ADD]]
 ; GCN: buffer_store_dword [[CONV]],
 define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %load = load i32, i32 addrspace(1)* %gep, align 4
   %add = add i32 %load, 2
   %inreg = and i32 %add, 255
@@ -186,7 +186,7 @@ define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias
 ; GCN-LABEL: {{^}}i8_zext_inreg_hi1_to_f32:
 define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %load = load i32, i32 addrspace(1)* %gep, align 4
   %inreg = and i32 %load, 65280
   %shr = lshr i32 %inreg, 8
@@ -200,7 +200,7 @@ define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias
 ; GCN-LABEL: {{^}}i8_zext_i32_to_f32:
 define amdgpu_kernel void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i8, i8 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i8, i8 addrspace(1)* %in, i32 %tid
   %load = load i8, i8 addrspace(1)* %gep, align 1
   %ext = zext i8 %load to i32
   %cvt = uitofp i32 %ext to float
@@ -211,7 +211,7 @@ define amdgpu_kernel void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out,
 ; GCN-LABEL: {{^}}v4i8_zext_v4i32_to_v4f32:
 define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
   %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 1
   %ext = zext <4 x i8> %load to <4 x i32>
   %cvt = uitofp <4 x i32> %ext to <4 x float>
@@ -226,7 +226,7 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* no
 ; GCN: buffer_store_dword [[CONV]]
 define amdgpu_kernel void @extract_byte0_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %val = load i32, i32 addrspace(1)* %gep
   %and = and i32 %val, 255
   %cvt = uitofp i32 %and to float
@@ -241,7 +241,7 @@ define amdgpu_kernel void @extract_byte0_to_f32(float addrspace(1)* noalias %out
 ; GCN: buffer_store_dword [[CONV]]
 define amdgpu_kernel void @extract_byte1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %val = load i32, i32 addrspace(1)* %gep
   %srl = lshr i32 %val, 8
   %and = and i32 %srl, 255
@@ -257,7 +257,7 @@ define amdgpu_kernel void @extract_byte1_to_f32(float addrspace(1)* noalias %out
 ; GCN: buffer_store_dword [[CONV]]
 define amdgpu_kernel void @extract_byte2_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %val = load i32, i32 addrspace(1)* %gep
   %srl = lshr i32 %val, 16
   %and = and i32 %srl, 255
@@ -273,7 +273,7 @@ define amdgpu_kernel void @extract_byte2_to_f32(float addrspace(1)* noalias %out
 ; GCN: buffer_store_dword [[CONV]]
 define amdgpu_kernel void @extract_byte3_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid 
+  %gep = getelementptr i32, i32 addrspace(1)* %in, i32 %tid
   %val = load i32, i32 addrspace(1)* %gep
   %srl = lshr i32 %val, 24
   %and = and i32 %srl, 255
diff --git a/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll b/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll
new file mode 100644
index 0000000000000..e7e7b9f907ca4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/dagcomb-shuffle-vecextend-non2.ll
@@ -0,0 +1,32 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; We are only checking that instruction selection can succeed in this case. This
+; cut down test results in no instructions, but that's fine.
+;
+; See https://llvm.org/PR33743 for details of the bug being addressed
+;
+; Checking that shufflevector with 3-vec mask is handled in
+; combineShuffleToVectorExtend
+;
+; GCN: s_endpgm
+
+define amdgpu_ps void @main(i32 %in1) local_unnamed_addr {
+.entry:
+  br i1 undef, label %bb12, label %bb
+
+bb:
+  %__llpc_global_proxy_r5.12.vec.insert = insertelement <4 x i32> undef, i32 %in1, i32 3
+  %tmp3 = shufflevector <4 x i32> %__llpc_global_proxy_r5.12.vec.insert, <4 x i32> undef, <3 x i32> <i32 undef, i32 undef, i32 1>
+  %tmp4 = bitcast <3 x i32> %tmp3 to <3 x float>
+  %a2.i123 = extractelement <3 x float> %tmp4, i32 2
+  %tmp5 = bitcast float %a2.i123 to i32
+  %__llpc_global_proxy_r2.0.vec.insert196 = insertelement <4 x i32> undef, i32 %tmp5, i32 0
+  br label %bb12
+
+bb12:
+  %__llpc_global_proxy_r2.0 = phi <4 x i32> [ %__llpc_global_proxy_r2.0.vec.insert196, %bb ], [ undef, %.entry ]
+  %tmp6 = shufflevector <4 x i32> %__llpc_global_proxy_r2.0, <4 x i32> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  %tmp7 = bitcast <3 x i32> %tmp6 to <3 x float>
+  %a0.i = extractelement <3 x float> %tmp7, i32 0
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/dead_copy.mir b/test/CodeGen/AMDGPU/dead_copy.mir
new file mode 100644
index 0000000000000..d581b29a19575
--- /dev/null
+++ b/test/CodeGen/AMDGPU/dead_copy.mir
@@ -0,0 +1,27 @@
+# RUN: llc -o - %s -march=amdgcn -mcpu=fiji  -run-pass=machine-cp -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: dead_copy
+# GCN:       bb.0
+# GCN-NOT:   dead %vgpr5 = COPY undef %vgpr11, implicit %exec
+# GCN:       %vgpr5 = COPY %vgpr11, implicit %exec
+
+---
+name: dead_copy
+
+body:    |
+
+  bb.0:
+    liveins: %vgpr11, %sgpr0, %sgpr1, %vgpr6, %vgpr7, %vgpr4
+
+    dead %vgpr5 = COPY undef %vgpr11, implicit %exec
+
+    %vgpr5 = COPY %vgpr11, implicit %exec
+
+    %sgpr14 = S_ADD_U32 %sgpr0, target-flags(amdgpu-gotprel) 1136, implicit-def %scc
+    %sgpr15 = S_ADDC_U32 %sgpr1, target-flags(amdgpu-gotprel32-lo) 0, implicit-def dead %scc, implicit %scc
+
+    %vgpr10 = COPY killed %sgpr14, implicit %exec
+    %vgpr11 = COPY killed %sgpr15, implicit %exec
+
+    FLAT_STORE_DWORDX4 %vgpr10_vgpr11, %vgpr4_vgpr5_vgpr6_vgpr7, 0, 0, 0, implicit %exec, implicit %flat_scr
+...
diff --git a/test/CodeGen/AMDGPU/debug-value.ll b/test/CodeGen/AMDGPU/debug-value.ll
new file mode 100644
index 0000000000000..30c134233b538
--- /dev/null
+++ b/test/CodeGen/AMDGPU/debug-value.ll
@@ -0,0 +1,106 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgizcl -verify-machineinstrs < %s | FileCheck %s
+
+%struct.wombat = type { [4 x i32], [4 x i32], [4 x i32] }
+
+define amdgpu_kernel void @wobble(i8 addrspace(1)* nocapture readonly %arg) #0 !dbg !4 {
+bb:
+  %tmp = load i32, i32 addrspace(1)* undef, align 4
+  %tmp1 = load <4 x float>, <4 x float> addrspace(1)* undef, align 16
+  %tmp2 = sext i32 %tmp to i64
+  %tmp3 = shufflevector <4 x float> undef, <4 x float> %tmp1, <2 x i32> <i32 3, i32 7>
+  %tmp4 = call float @barney() #2
+  %tmp5 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 0
+  %tmp6 = bitcast i8 addrspace(1)* %tmp5 to <2 x float> addrspace(1)*
+  %tmp7 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 0
+  %tmp8 = bitcast i8 addrspace(1)* %tmp7 to %struct.wombat addrspace(1)*
+  %tmp9 = getelementptr inbounds %struct.wombat, %struct.wombat addrspace(1)* %tmp8, i64 %tmp2, i32 2, i64 0
+  %tmp10 = load i32, i32 addrspace(1)* %tmp9, align 4
+  %tmp11 = sext i32 %tmp10 to i64
+  %tmp12 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %tmp6, i64 %tmp11
+  %tmp13 = bitcast <2 x float> addrspace(1)* %tmp12 to i64 addrspace(1)*
+  %tmp14 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 undef
+  %tmp15 = bitcast i8 addrspace(1)* %tmp14 to <4 x float> addrspace(1)*
+  %tmp16 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %tmp15, i64 undef
+  %tmp17 = load <4 x float>, <4 x float> addrspace(1)* %tmp16, align 16
+  %tmp18 = fsub <4 x float> undef, %tmp17
+  %tmp19 = fadd float undef, 0.000000e+00
+  %tmp20 = fcmp oeq float %tmp19, 0.000000e+00
+  br i1 %tmp20, label %bb21, label %bb25
+
+bb21:                                             ; preds = %bb
+  %tmp22 = fmul <4 x float> %tmp18, undef
+  %tmp23 = fadd <4 x float> undef, %tmp22
+  %tmp24 = fmul <4 x float> undef, undef
+  br label %bb28
+
+bb25:                                             ; preds = %bb
+  %tmp26 = insertelement <4 x float> undef, float 0.000000e+00, i32 1
+  %tmp27 = insertelement <4 x float> %tmp26, float undef, i32 2
+  br label %bb28
+
+bb28:                                             ; preds = %bb25, %bb21
+  %tmp29 = phi <4 x float> [ %tmp27, %bb25 ], [ %tmp24, %bb21 ]
+  store <4 x float> %tmp29, <4 x float> addrspace(5)* undef, align 16
+  %tmp30 = getelementptr inbounds %struct.wombat, %struct.wombat addrspace(1)* %tmp8, i64 %tmp2, i32 2, i64 2
+  %tmp31 = load i32, i32 addrspace(1)* %tmp30, align 4
+  %tmp32 = sext i32 %tmp31 to i64
+  %tmp33 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %tmp6, i64 %tmp32
+  %tmp34 = bitcast <2 x float> addrspace(1)* %tmp33 to i64 addrspace(1)*
+  %tmp35 = load i64, i64 addrspace(1)* %tmp34, align 8
+  %tmp36 = load i32, i32 addrspace(1)* undef, align 4
+  %tmp37 = sext i32 %tmp36 to i64
+  %tmp38 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* null, i64 %tmp37
+  %tmp39 = load <4 x float>, <4 x float> addrspace(1)* %tmp38, align 16
+  %tmp40 = load <4 x float>, <4 x float> addrspace(1)* undef, align 16
+  %tmp41 = fsub <4 x float> zeroinitializer, %tmp40
+  %tmp42 = fsub <4 x float> %tmp39, %tmp40
+  %tmp43 = extractelement <4 x float> %tmp40, i32 1
+  %tmp44 = fsub float %tmp43, undef
+  %tmp45 = fadd float undef, undef
+  %tmp46 = fdiv float %tmp44, %tmp45
+  %tmp47 = insertelement <4 x float> undef, float %tmp46, i32 0
+  %tmp48 = shufflevector <4 x float> %tmp47, <4 x float> undef, <4 x i32> zeroinitializer
+  %tmp49 = fsub <4 x float> %tmp48, %tmp40
+  %tmp50 = extractelement <4 x float> %tmp41, i32 1
+  %tmp51 = extractelement <4 x float> %tmp42, i32 2
+  %tmp52 = fmul float undef, undef
+  %tmp53 = fadd float %tmp52, undef
+  %tmp54 = fadd float %tmp51, %tmp53
+  %tmp55 = extractelement <4 x float> %tmp49, i32 1
+  %tmp56 = fmul float %tmp55, %tmp50
+  %tmp57 = fmul float %tmp54, %tmp56
+  %tmp58 = fdiv float %tmp57, 0.000000e+00
+  ; CHECK: ;DEBUG_VALUE: foo:var <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef]
+  call void @llvm.dbg.value(metadata <4 x float> %tmp29, metadata !3, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)) #2, !dbg !5
+  %tmp59 = bitcast i64 %tmp35 to <2 x float>
+  %tmp60 = insertelement <2 x float> undef, float %tmp58, i32 0
+  %tmp61 = shufflevector <2 x float> %tmp60, <2 x float> undef, <2 x i32> zeroinitializer
+  %tmp62 = fmul <2 x float> %tmp61, undef
+  %tmp63 = fsub <2 x float> %tmp62, %tmp59
+  %tmp64 = extractelement <2 x float> %tmp63, i64 0
+  call void @eggs(float %tmp64) #2
+  store <2 x float> %tmp3, <2 x float> addrspace(1)* undef, align 8
+  store float 0.000000e+00, float addrspace(1)* undef, align 4
+  ret void
+}
+
+declare float @barney() #2
+declare void @eggs(float) #2
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { convergent nounwind "target-cpu"="gfx900" "target-features"="+fp32-denormals" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "foo.cl", directory: "/tmp")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocalVariable(name: "var", arg: 8, scope: !4)
+!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, type: !12, isLocal: false, isDefinition: true, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+!5 = !DILocation(line: 69, scope: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
diff --git a/test/CodeGen/AMDGPU/debugger-emit-prologue.ll b/test/CodeGen/AMDGPU/debugger-emit-prologue.ll
index 734905ba2b082..db40ea6a41bc1 100644
--- a/test/CodeGen/AMDGPU/debugger-emit-prologue.ll
+++ b/test/CodeGen/AMDGPU/debugger-emit-prologue.ll
@@ -1,5 +1,6 @@
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=+amdgpu-debugger-emit-prologue -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s --check-prefix=NOATTR
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=fiji -mattr=+amdgpu-debugger-emit-prologue -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s --check-prefix=NOATTR
+target datalayout = "A5"
 
 ; CHECK: debug_wavefront_private_segment_offset_sgpr = [[SOFF:[0-9]+]]
 ; CHECK: debug_private_segment_buffer_sgpr = [[SREG:[0-9]+]]
@@ -25,16 +26,16 @@
 ; Function Attrs: nounwind
 define amdgpu_kernel void @test(i32 addrspace(1)* %A) #0 !dbg !12 {
 entry:
-  %A.addr = alloca i32 addrspace(1)*, align 4
-  store i32 addrspace(1)* %A, i32 addrspace(1)** %A.addr, align 4
-  call void @llvm.dbg.declare(metadata i32 addrspace(1)** %A.addr, metadata !17, metadata !18), !dbg !19
-  %0 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !20
+  %A.addr = alloca i32 addrspace(1)*, align 4, addrspace(5)
+  store i32 addrspace(1)* %A, i32 addrspace(1)* addrspace(5)* %A.addr, align 4
+  call void @llvm.dbg.declare(metadata i32 addrspace(1)* addrspace(5)* %A.addr, metadata !17, metadata !18), !dbg !19
+  %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !20
   %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %0, i32 0, !dbg !20
   store i32 1, i32 addrspace(1)* %arrayidx, align 4, !dbg !21
-  %1 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !22
+  %1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !22
   %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %1, i32 1, !dbg !22
   store i32 2, i32 addrspace(1)* %arrayidx1, align 4, !dbg !23
-  %2 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !24
+  %2 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !24
   %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %2, i32 2, !dbg !24
   store i32 3, i32 addrspace(1)* %arrayidx2, align 4, !dbg !25
   ret void, !dbg !26
@@ -57,8 +58,8 @@ attributes #1 = { nounwind readnone }
 !3 = !{void (i32 addrspace(1)*)* @test, !4, !5, !6, !7, !8}
 !4 = !{!"kernel_arg_addr_space", i32 1}
 !5 = !{!"kernel_arg_access_qual", !"none"}
-!6 = !{!"kernel_arg_type", !"int*"}
-!7 = !{!"kernel_arg_base_type", !"int*"}
+!6 = !{!"kernel_arg_type", !"int addrspace(5)*"}
+!7 = !{!"kernel_arg_base_type", !"int addrspace(5)*"}
 !8 = !{!"kernel_arg_type_qual", !""}
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/AMDGPU/debugger-insert-nops.ll b/test/CodeGen/AMDGPU/debugger-insert-nops.ll
index fcdbfb10a8ca4..5479091233014 100644
--- a/test/CodeGen/AMDGPU/debugger-insert-nops.ll
+++ b/test/CodeGen/AMDGPU/debugger-insert-nops.ll
@@ -1,5 +1,6 @@
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=+amdgpu-debugger-insert-nops -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=+amdgpu-debugger-insert-nops -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECKNOP
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=fiji -mattr=+amdgpu-debugger-insert-nops -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=fiji -mattr=+amdgpu-debugger-insert-nops -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECKNOP
+target datalayout = "A5"
 
 ; This test expects that we have one instance for each line in some order with "s_nop 0" instances after each.
 
@@ -24,16 +25,16 @@
 ; Function Attrs: nounwind
 define amdgpu_kernel void @test(i32 addrspace(1)* %A) #0 !dbg !12 {
 entry:
-  %A.addr = alloca i32 addrspace(1)*, align 4
-  store i32 addrspace(1)* %A, i32 addrspace(1)** %A.addr, align 4
-  call void @llvm.dbg.declare(metadata i32 addrspace(1)** %A.addr, metadata !17, metadata !18), !dbg !19
-  %0 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !20
+  %A.addr = alloca i32 addrspace(1)*, align 4, addrspace(5)
+  store i32 addrspace(1)* %A, i32 addrspace(1)* addrspace(5)* %A.addr, align 4
+  call void @llvm.dbg.declare(metadata i32 addrspace(1)* addrspace(5)* %A.addr, metadata !17, metadata !18), !dbg !19
+  %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !20
   %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %0, i32 0, !dbg !20
   store i32 1, i32 addrspace(1)* %arrayidx, align 4, !dbg !20
-  %1 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !22
+  %1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !22
   %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %1, i32 1, !dbg !22
   store i32 2, i32 addrspace(1)* %arrayidx1, align 4, !dbg !23
-  %2 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !24
+  %2 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !24
   %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %2, i32 2, !dbg !24
   store i32 3, i32 addrspace(1)* %arrayidx2, align 4, !dbg !25
   ret void, !dbg !26
@@ -56,8 +57,8 @@ attributes #1 = { nounwind readnone }
 !3 = !{void (i32 addrspace(1)*)* @test, !4, !5, !6, !7, !8}
 !4 = !{!"kernel_arg_addr_space", i32 1}
 !5 = !{!"kernel_arg_access_qual", !"none"}
-!6 = !{!"kernel_arg_type", !"int*"}
-!7 = !{!"kernel_arg_base_type", !"int*"}
+!6 = !{!"kernel_arg_type", !"int addrspace(5)*"}
+!7 = !{!"kernel_arg_base_type", !"int addrspace(5)*"}
 !8 = !{!"kernel_arg_type_qual", !""}
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/AMDGPU/debugger-reserve-regs.ll b/test/CodeGen/AMDGPU/debugger-reserve-regs.ll
index 764c60b12bf9f..ae40ce3b7924f 100644
--- a/test/CodeGen/AMDGPU/debugger-reserve-regs.ll
+++ b/test/CodeGen/AMDGPU/debugger-reserve-regs.ll
@@ -1,5 +1,6 @@
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=+amdgpu-debugger-reserve-regs -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=gfx901 -mattr=+amdgpu-debugger-reserve-regs -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=fiji -mattr=+amdgpu-debugger-reserve-regs -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa-amdgiz -mcpu=gfx901 -mattr=+amdgpu-debugger-reserve-regs -verify-machineinstrs < %s | FileCheck %s
+target datalayout = "A5"
 ; CHECK: reserved_vgpr_first = {{[0-9]+}}
 ; CHECK-NEXT: reserved_vgpr_count = 4
 ; CHECK: ReservedVGPRFirst: {{[0-9]+}}
@@ -8,16 +9,16 @@
 ; Function Attrs: nounwind
 define amdgpu_kernel void @test(i32 addrspace(1)* %A) #0 !dbg !12 {
 entry:
-  %A.addr = alloca i32 addrspace(1)*, align 4
-  store i32 addrspace(1)* %A, i32 addrspace(1)** %A.addr, align 4
-  call void @llvm.dbg.declare(metadata i32 addrspace(1)** %A.addr, metadata !17, metadata !18), !dbg !19
-  %0 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !20
+  %A.addr = alloca i32 addrspace(1)*, align 4, addrspace(5)
+  store i32 addrspace(1)* %A, i32 addrspace(1)* addrspace(5)* %A.addr, align 4
+  call void @llvm.dbg.declare(metadata i32 addrspace(1)* addrspace(5)* %A.addr, metadata !17, metadata !18), !dbg !19
+  %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !20
   %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %0, i32 0, !dbg !20
   store i32 1, i32 addrspace(1)* %arrayidx, align 4, !dbg !21
-  %1 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !22
+  %1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !22
   %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %1, i32 1, !dbg !22
   store i32 2, i32 addrspace(1)* %arrayidx1, align 4, !dbg !23
-  %2 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !24
+  %2 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !24
   %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %2, i32 2, !dbg !24
   store i32 3, i32 addrspace(1)* %arrayidx2, align 4, !dbg !25
   ret void, !dbg !26
@@ -40,8 +41,8 @@ attributes #1 = { nounwind readnone }
 !3 = !{void (i32 addrspace(1)*)* @test, !4, !5, !6, !7, !8}
 !4 = !{!"kernel_arg_addr_space", i32 1}
 !5 = !{!"kernel_arg_access_qual", !"none"}
-!6 = !{!"kernel_arg_type", !"int*"}
-!7 = !{!"kernel_arg_base_type", !"int*"}
+!6 = !{!"kernel_arg_type", !"int addrspace(5)*"}
+!7 = !{!"kernel_arg_base_type", !"int addrspace(5)*"}
 !8 = !{!"kernel_arg_type_qual", !""}
 !9 = !{i32 2, !"Dwarf Version", i32 2}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/test/CodeGen/AMDGPU/detect-dead-lanes.mir b/test/CodeGen/AMDGPU/detect-dead-lanes.mir
index c265b8e2ad2ea..12460d25f3b20 100644
--- a/test/CodeGen/AMDGPU/detect-dead-lanes.mir
+++ b/test/CodeGen/AMDGPU/detect-dead-lanes.mir
@@ -6,12 +6,12 @@
 # CHECK: S_NOP 0, implicit-def %0
 # CHECK: S_NOP 0, implicit-def %1
 # CHECK: S_NOP 0, implicit-def dead %2
-# CHECK: %3 = REG_SEQUENCE %0, {{[0-9]+}}, %1, {{[0-9]+}}, undef %2, {{[0-9]+}}
+# CHECK: %3:sreg_128 = REG_SEQUENCE %0,  %subreg.sub0, %1,  %subreg.sub1, undef %2, %subreg.sub3
 # CHECK: S_NOP 0, implicit %3.sub0
 # CHECK: S_NOP 0, implicit %3.sub1
 # CHECK: S_NOP 0, implicit undef %3.sub2
-# CHECK: %4 = COPY %3.sub0_sub1
-# CHECK: %5 = COPY undef %3.sub2_sub3
+# CHECK: %4:sreg_64 = COPY %3.sub0_sub1
+# CHECK: %5:sreg_64 = COPY undef %3.sub2_sub3
 # CHECK: S_NOP 0, implicit %4.sub0
 # CHECK: S_NOP 0, implicit %4.sub1
 # CHECK: S_NOP 0, implicit undef %5.sub0
@@ -42,9 +42,9 @@ body: |
 # Check defined lanes transfer; Includes checking for some special cases like
 # undef operands or IMPLICIT_DEF definitions.
 # CHECK-LABEL: name: test1
-# CHECK: %0 = REG_SEQUENCE %sgpr0, {{[0-9]+}}, %sgpr0, {{[0-9]+}}
-# CHECK: %1 = INSERT_SUBREG %0, %sgpr1, {{[0-9]+}}
-# CHECK: %2 = INSERT_SUBREG %0.sub2_sub3, %sgpr42, {{[0-9]+}}
+# CHECK: %0:sreg_128 = REG_SEQUENCE %sgpr0, %subreg.sub0, %sgpr0, %subreg.sub2
+# CHECK: %1:sreg_128 = INSERT_SUBREG %0, %sgpr1,  %subreg.sub3
+# CHECK: %2:sreg_64 = INSERT_SUBREG %0.sub2_sub3, %sgpr42,  %subreg.sub0
 # CHECK: S_NOP 0, implicit %1.sub0
 # CHECK: S_NOP 0, implicit undef %1.sub1
 # CHECK: S_NOP 0, implicit %1.sub2
@@ -52,25 +52,25 @@ body: |
 # CHECK: S_NOP 0, implicit %2.sub0
 # CHECK: S_NOP 0, implicit undef %2.sub1
 
-# CHECK: %3 = IMPLICIT_DEF
-# CHECK: %4 = INSERT_SUBREG %0, undef %3, {{[0-9]+}}
+# CHECK: %3:sreg_32_xm0 = IMPLICIT_DEF
+# CHECK: %4:sreg_128 = INSERT_SUBREG %0, undef %3, %subreg.sub0
 # CHECK: S_NOP 0, implicit undef %4.sub0
 # CHECK: S_NOP 0, implicit undef %4.sub1
 # CHECK: S_NOP 0, implicit %4.sub2
 # CHECK: S_NOP 0, implicit undef %4.sub3
 
-# CHECK: %5 = EXTRACT_SUBREG %0, {{[0-9]+}}
-# CHECK: %6 = EXTRACT_SUBREG %5, {{[0-9]+}}
-# CHECK: %7 = EXTRACT_SUBREG %5, {{[0-9]+}}
+# CHECK: %5:sreg_64 = EXTRACT_SUBREG %0, %subreg.sub0_sub1
+# CHECK: %6:sreg_32_xm0 = EXTRACT_SUBREG %5, %subreg.sub0
+# CHECK: %7:sreg_32_xm0 = EXTRACT_SUBREG %5, %subreg.sub1
 # CHECK: S_NOP 0, implicit %5
 # CHECK: S_NOP 0, implicit %6
 # CHECK: S_NOP 0, implicit undef %7
 
-# CHECK: %8 = IMPLICIT_DEF
-# CHECK: %9 = EXTRACT_SUBREG undef %8, {{[0-9]+}}
+# CHECK: %8:sreg_64 = IMPLICIT_DEF
+# CHECK: %9:sreg_32_xm0 = EXTRACT_SUBREG undef %8, %subreg.sub1
 # CHECK: S_NOP 0, implicit undef %9
 
-# CHECK: %10 = EXTRACT_SUBREG undef %0, {{[0-9]+}}
+# CHECK: %10:sreg_128 = EXTRACT_SUBREG undef %0, %subreg.sub2_sub3
 # CHECK: S_NOP 0, implicit undef %10
 name: test1
 registers:
@@ -125,29 +125,29 @@ body: |
 # CHECK: S_NOP 0, implicit-def dead %0
 # CHECK: S_NOP 0, implicit-def %1
 # CHECK: S_NOP 0, implicit-def %2
-# CHECK: %3 = REG_SEQUENCE undef %0, {{[0-9]+}}, %1, {{[0-9]+}}, %2, {{[0-9]+}}
+# CHECK: %3:sreg_128 = REG_SEQUENCE undef %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2_sub3
 # CHECK: S_NOP 0, implicit %3.sub1
 # CHECK: S_NOP 0, implicit %3.sub3
 
 # CHECK: S_NOP 0, implicit-def %4
 # CHECK: S_NOP 0, implicit-def dead %5
-# CHECK: %6 = REG_SEQUENCE %4, {{[0-9]+}}, undef %5, {{[0-9]+}}
+# CHECK: %6:sreg_64 = REG_SEQUENCE %4, %subreg.sub0, undef %5, %subreg.sub1
 # CHECK: S_NOP 0, implicit %6
 
 # CHECK: S_NOP 0, implicit-def dead %7
 # CHECK: S_NOP 0, implicit-def %8
-# CHECK: %9 = INSERT_SUBREG undef %7, %8, {{[0-9]+}}
+# CHECK: %9:sreg_128 = INSERT_SUBREG undef %7, %8, %subreg.sub2_sub3
 # CHECK: S_NOP 0, implicit %9.sub2
 
 # CHECK: S_NOP 0, implicit-def %10
 # CHECK: S_NOP 0, implicit-def dead %11
-# CHECK: %12 = INSERT_SUBREG %10, undef %11, {{[0-9]+}}
+# CHECK: %12:sreg_128 = INSERT_SUBREG %10, undef %11, %subreg.sub0_sub1
 # CHECK: S_NOP 0, implicit %12.sub3
 
 # CHECK: S_NOP 0, implicit-def %13
 # CHECK: S_NOP 0, implicit-def dead %14
-# CHECK: %15 = REG_SEQUENCE %13, {{[0-9]+}}, undef %14, {{[0-9]+}}
-# CHECK: %16 = EXTRACT_SUBREG %15, {{[0-9]+}}
+# CHECK: %15:sreg_128 = REG_SEQUENCE %13, %subreg.sub0_sub1, undef %14, %subreg.sub2_sub3
+# CHECK: %16:sreg_64 = EXTRACT_SUBREG %15, %subreg.sub0_sub1
 # CHECK: S_NOP 0, implicit %16.sub1
 
 name: test2
@@ -205,7 +205,7 @@ body: |
 # CHECK-LABEL: name: test3
 # CHECK: S_NOP 0, implicit-def %0
 # CHECK: %vcc = COPY %0
-# CHECK: %1 = COPY %vcc
+# CHECK: %1:sreg_64 = COPY %vcc
 # CHECK: S_NOP 0, implicit %1
 name: test3
 tracksRegLiveness: true
@@ -225,7 +225,7 @@ body: |
 # CHECK-LABEL: name: test4
 # CHECK: S_NOP 0, implicit-def dead %0
 # CHECK: KILL undef %0
-# CHECK: %1 = IMPLICIT_DEF
+# CHECK: %1:sreg_64 = IMPLICIT_DEF
 # CHECK: S_NOP 0, implicit undef %1
 name: test4
 tracksRegLiveness: true
@@ -245,7 +245,7 @@ body: |
 # used.
 # CHECK-LABEL: name: test5
 # CHECK: S_NOP 0, implicit-def %0
-# CHECK: %1 = REG_SEQUENCE undef %0, {{[0-9]+}}, %0, {{[0-9]+}}
+# CHECK: %1:sreg_64 = REG_SEQUENCE undef %0, %subreg.sub0, %0, %subreg.sub1
 # CHECK: S_NOP 0, implicit %1.sub1
 name: test5
 tracksRegLiveness: true
@@ -265,10 +265,10 @@ body: |
 # CHECK: S_NOP 0, implicit-def %0
 # CHECK: S_NOP 0, implicit-def dead %1
 # CHECK: S_NOP 0, implicit-def dead %2
-# CHECK: %3 = REG_SEQUENCE %0, {{[0-9]+}}, undef %1, {{[0-9]+}}, undef %2, {{[0-9]+}}
+# CHECK: %3:sreg_128 = REG_SEQUENCE %0,  %subreg.sub0, undef %1,  %subreg.sub1, undef %2,  %subreg.sub2
 
 # CHECK: bb.1:
-# CHECK: %4 = PHI %3, %bb.0, %5, %bb.1
+# CHECK: %4:sreg_128 = PHI %3, %bb.0, %5, %bb.1
 
 # CHECK: bb.2:
 # CHECK:   S_NOP 0, implicit %4.sub0
@@ -315,12 +315,12 @@ body: |
 # CHECK: S_NOP 0, implicit-def %1
 # CHECK: S_NOP 0, implicit-def dead %2
 # CHECK: S_NOP 0, implicit-def %3
-# CHECK: %4 = REG_SEQUENCE %0, {{[0-9]+}}, %1, {{[0-9]+}}, undef %2, {{[0-9]+}}, %3, {{[0-9]+}}
+# CHECK: %4:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, undef %2, %subreg.sub2, %3, %subreg.sub3
 
 # CHECK: bb.1:
-# CHECK: %5 = PHI %4, %bb.0, %6, %bb.1
+# CHECK: %5:sreg_128 = PHI %4, %bb.0, %6, %bb.1
 
-# CHECK: %6 = REG_SEQUENCE %5.sub1, {{[0-9]+}}, %5.sub3, {{[0-9]+}}, undef %5.sub2, {{[0-9]+}}, %5.sub0, {{[0-9]+}}
+# CHECK: %6:sreg_128 = REG_SEQUENCE %5.sub1, %subreg.sub0, %5.sub3, %subreg.sub1, undef %5.sub2, %subreg.sub2, %5.sub0, %subreg.sub3
 
 # CHECK: bb.2:
 # CHECK:   S_NOP 0, implicit %6.sub3
@@ -361,12 +361,12 @@ body: |
 # CHECK-LABEL: name: loop2
 # CHECK: bb.0:
 # CHECK: S_NOP 0, implicit-def %0
-# CHECK: %1 = REG_SEQUENCE %0, {{[0-9]+}}
+# CHECK: %1:sreg_128 = REG_SEQUENCE %0, %subreg.sub0
 
 # CHECK: bb.1:
-# CHECK: %2 = PHI %1, %bb.0, %3, %bb.1
+# CHECK: %2:sreg_128 = PHI %1, %bb.0, %3, %bb.1
 
-# CHECK: %3 = REG_SEQUENCE %2.sub3, {{[0-9]+}}, undef %2.sub1, {{[0-9]+}}, %2.sub0, {{[0-9]+}}, %2.sub2, {{[0-9]+}}
+# CHECK: %3:sreg_128 = REG_SEQUENCE %2.sub3, %subreg.sub0, undef %2.sub1, %subreg.sub1, %2.sub0, %subreg.sub2, %2.sub2, %subreg.sub3
 
 # CHECK: bb.2:
 # CHECK:   S_NOP 0, implicit %2.sub0
diff --git a/test/CodeGen/AMDGPU/ds-combine-large-stride.ll b/test/CodeGen/AMDGPU/ds-combine-large-stride.ll
index a723b0210adeb..7dd1f90e91410 100644
--- a/test/CodeGen/AMDGPU/ds-combine-large-stride.ll
+++ b/test/CodeGen/AMDGPU/ds-combine-large-stride.ll
@@ -1,15 +1,18 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; GCN-LABEL: ds_read32_combine_stride_400:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x320, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x640, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
@@ -46,12 +49,15 @@ bb:
 ; GCN-LABEL: ds_read32_combine_stride_400_back:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x320, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x640, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:100
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:100
 ; GCN-DAG: ds_read2_b32  v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:100
@@ -124,12 +130,15 @@ bb:
 ; GCN-LABEL: ds_read32_combine_stride_8192_shifted:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 8, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x4008, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x8008, [[BASE]]
+
 ; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:32
 ; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:32
 ; GCN-DAG: ds_read2st64_b32 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:32
@@ -160,8 +169,10 @@ bb:
 ; GCN-LABEL: ds_read64_combine_stride_400:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset1:50
 ; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:100 offset1:150
 ; GCN-DAG: ds_read2_b64  v[{{[0-9]+:[0-9]+}}], [[BASE]] offset0:200 offset1:250
@@ -198,12 +209,15 @@ bb:
 ; GCN-LABEL: ds_read64_combine_stride_8192_shifted:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 8, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x4008, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x8008, [[BASE]]
+
 ; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B1]] offset1:16
 ; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B2]] offset1:16
 ; GCN-DAG: ds_read2st64_b64 v[{{[0-9]+:[0-9]+}}], [[B3]] offset1:16
@@ -234,12 +248,15 @@ bb:
 ; GCN-LABEL: ds_write32_combine_stride_400:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x320, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x640, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
 ; GCN-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
 ; GCN-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
@@ -267,12 +284,15 @@ bb:
 ; GCN-LABEL: ds_write32_combine_stride_400_back:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x320, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x640, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x320, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x640, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_write2_b32 [[BASE]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
 ; GCN-DAG: ds_write2_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
 ; GCN-DAG: ds_write2_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:100
@@ -327,12 +347,15 @@ bb:
 ; GCN-LABEL: ds_write32_combine_stride_8192_shifted:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 4, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x4004, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x8004, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 4, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x4004, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x8004, [[BASE]]
+
 ; GCN-DAG: ds_write2st64_b32 [[B1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
 ; GCN-DAG: ds_write2st64_b32 [[B2]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
 ; GCN-DAG: ds_write2st64_b32 [[B3]], v{{[0-9]+}}, v{{[0-9]+}} offset1:32
@@ -356,8 +379,10 @@ bb:
 ; GCN-LABEL: ds_write64_combine_stride_400:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 0x960, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 0x960, [[BASE]]
+
 ; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:50
 ; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:100 offset1:150
 ; GCN-DAG: ds_write2_b64 [[BASE]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset0:200 offset1:250
@@ -385,12 +410,15 @@ bb:
 ; GCN-LABEL: ds_write64_combine_stride_8192_shifted:
 ; GCN:     s_load_dword [[ARG:s[0-9]+]], s[4:5], 0x0
 ; GCN:     v_mov_b32_e32 [[BASE:v[0-9]+]], [[ARG]]
-; GCN-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GCN-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
-; GFX9-DAG: v_add_i32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; VI-DAG: v_add_u32_e32 [[B1:v[0-9]+]], vcc, 8, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B2:v[0-9]+]], vcc, 0x4008, [[BASE]]
+; VI-DAG: v_add_u32_e32 [[B3:v[0-9]+]], vcc, 0x8008, [[BASE]]
+
+; GFX9-DAG: v_add_u32_e32 [[B1:v[0-9]+]], 8, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B2:v[0-9]+]], 0x4008, [[BASE]]
+; GFX9-DAG: v_add_u32_e32 [[B3:v[0-9]+]], 0x8008, [[BASE]]
+
 ; GCN-DAG: ds_write2st64_b64 [[B1]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
 ; GCN-DAG: ds_write2st64_b64 [[B2]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
 ; GCN-DAG: ds_write2st64_b64 [[B3]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] offset1:16
diff --git a/test/CodeGen/AMDGPU/ds-sub-offset.ll b/test/CodeGen/AMDGPU/ds-sub-offset.ll
index d74bd5aa15ac1..05d8dc0b9a045 100644
--- a/test/CodeGen/AMDGPU/ds-sub-offset.ll
+++ b/test/CodeGen/AMDGPU/ds-sub-offset.ll
@@ -1,4 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
@@ -6,7 +7,8 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 ; GCN-LABEL: {{^}}write_ds_sub0_offset0_global:
 ; GCN: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v0
-; GCN: v_sub_i32_e32 [[BASEPTR:v[0-9]+]], vcc, 0, [[SHL]]
+; CI: v_sub_i32_e32 [[BASEPTR:v[0-9]+]], vcc, 0, [[SHL]]
+; GFX9: v_sub_u32_e32 [[BASEPTR:v[0-9]+]], 0, [[SHL]]
 ; GCN: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b
 ; GCN: ds_write_b32 [[BASEPTR]], [[VAL]] offset:12
 define amdgpu_kernel void @write_ds_sub0_offset0_global() #0 {
@@ -21,7 +23,8 @@ entry:
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
 ; GCN: ds_write_b8 [[NEG]], [[K]] offset:65535
 define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 {
@@ -36,7 +39,8 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset() #1 {
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_max_offset_p1:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x10000, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x10000, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x10000, [[SCALED]]
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
 ; GCN: ds_write_b8 [[NEG]], [[K]]{{$}}
 define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 {
@@ -51,7 +55,8 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_max_offset_p1() #1 {
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
 ; GCN-NOT: v_sub
 ; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}}
@@ -73,7 +78,8 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use() #1 {
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_multi_use_same_offset:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 13
 ; GCN-NOT: v_sub
 ; GCN: ds_write_b32 [[NEG]], [[K]] offset:123{{$}}
@@ -93,7 +99,8 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_multi_use_same_offset() #1 {
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SCALED]]
 ; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset0:254 offset1:255
 define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 {
   %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
@@ -107,7 +114,8 @@ define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset() #1 {
 
 ; GCN-LABEL: {{^}}add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1:
 ; GCN-DAG: v_lshlrev_b32_e32 [[SCALED:v[0-9]+]], 2, v0
-; GCN-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x3fc, [[SCALED]]
+; CI-DAG: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0x3fc, [[SCALED]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG:v[0-9]+]], 0x3fc, [[SCALED]]
 ; GCN: ds_write2_b32 [[NEG]], {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}}
 define amdgpu_kernel void @add_x_shl_neg_to_sub_misaligned_i64_max_offset_p1() #1 {
   %x.i = call i32 @llvm.amdgcn.workitem.id.x() #0
diff --git a/test/CodeGen/AMDGPU/ds_read2.ll b/test/CodeGen/AMDGPU/ds_read2.ll
index deb90df99dcf4..131afb0c6aec6 100644
--- a/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/test/CodeGen/AMDGPU/ds_read2.ll
@@ -1,4 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9 %s
 
 ; FIXME: We don't get cases where the address was an SGPR because we
 ; get a copy to the address register for each one.
@@ -6,12 +7,16 @@
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.f64 = addrspace(3) global [512 x double] undef, align 8
 
-; SI-LABEL: @simple_read2_f32
-; SI: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: {{^}}simple_read2_f32:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
+; CI: buffer_store_dword [[RESULT]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -25,12 +30,16 @@ define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 {
   ret void
 }
 
-; SI-LABEL: @simple_read2_f32_max_offset
-; SI: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: {{^}}simple_read2_f32_max_offset:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
+
+; CI: buffer_store_dword [[RESULT]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -44,11 +53,14 @@ define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out)
   ret void
 }
 
-; SI-LABEL: @simple_read2_f32_too_far
-; SI-NOT ds_read2_b32
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_too_far
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT ds_read2_b32
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -62,10 +74,13 @@ define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0
   ret void
 }
 
-; SI-LABEL: @simple_read2_f32_x2
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_x2
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 0
@@ -93,11 +108,14 @@ define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 {
 }
 
 ; Make sure there is an instruction between the two sets of reads.
-; SI-LABEL: @simple_read2_f32_x2_barrier
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
-; SI: s_barrier
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_x2_barrier
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8
+; GCN: s_barrier
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 0
@@ -129,10 +147,13 @@ define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out)
 ; For some reason adding something to the base address for the first
 ; element results in only folding the inner pair.
 
-; SI-LABEL: @simple_read2_f32_x2_nonzero_base
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_x2_nonzero_base
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -165,11 +186,14 @@ define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)*
 ; Base pointers come from different subregister of same super
 ; register. We can't safely merge this.
 
-; SI-LABEL: @read2_ptr_is_subreg_arg_f32
-; SI-NOT: ds_read2_b32
-; SI: ds_read_b32
-; SI: ds_read_b32
-; SI: s_endpgm
+; GCN-LABEL: @read2_ptr_is_subreg_arg_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2_b32
+; GCN: ds_read_b32
+; GCN: ds_read_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
@@ -191,11 +215,14 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out,
 ; sure we are really rejecting it because of the different
 ; subregisters.
 
-; SI-LABEL: @read2_ptr_is_subreg_arg_offset_f32
-; SI-NOT: ds_read2_b32
-; SI: ds_read_b32
-; SI: ds_read_b32
-; SI: s_endpgm
+; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2_b32
+; GCN: ds_read_b32
+; GCN: ds_read_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0
@@ -216,9 +243,12 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)
   ret void
 }
 
-; SI-LABEL: {{^}}read2_ptr_is_subreg_f32:
-; SI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}}
-; SI: s_endpgm
+; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}}
+; GCN: s_endpgm
 define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0
@@ -238,11 +268,14 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0
   ret void
 }
 
-; SI-LABEL: @simple_read2_f32_volatile_0
-; SI-NOT ds_read2_b32
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_volatile_0
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT ds_read2_b32
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -256,11 +289,14 @@ define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out)
   ret void
 }
 
-; SI-LABEL: @simple_read2_f32_volatile_1
-; SI-NOT ds_read2_b32
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f32_volatile_1
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT ds_read2_b32
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -277,9 +313,12 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out)
 ; Can't fold since not correctly aligned.
 ; XXX: This isn't really testing anything useful now. I think CI
 ; allows unaligned LDS accesses, which would be a problem here.
-; SI-LABEL: @unaligned_read2_f32
-; SI-NOT: ds_read2_b32
-; SI: s_endpgm
+; GCN-LABEL: @unaligned_read2_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i
@@ -293,9 +332,12 @@ define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float a
   ret void
 }
 
-; SI-LABEL: @misaligned_2_simple_read2_f32
-; SI-NOT: ds_read2_b32
-; SI: s_endpgm
+; GCN-LABEL: @misaligned_2_simple_read2_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i
@@ -309,12 +351,16 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %ou
   ret void
 }
 
-; SI-LABEL: @simple_read2_f64
-; SI: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, {{v[0-9]+}}
-; SI: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, {{v[0-9]+}}
+; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
+
+; CI: buffer_store_dwordx2 [[RESULT]]
+; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
@@ -328,9 +374,12 @@ define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 {
   ret void
 }
 
-; SI-LABEL: @simple_read2_f64_max_offset
-; SI: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f64_max_offset
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
@@ -344,11 +393,14 @@ define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out
   ret void
 }
 
-; SI-LABEL: @simple_read2_f64_too_far
-; SI-NOT ds_read2_b64
-; SI: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
-; SI: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2_f64_too_far
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT ds_read2_b64
+; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
@@ -363,10 +415,13 @@ define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #
 }
 
 ; Alignment only 4
-; SI-LABEL: @misaligned_read2_f64
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15
-; SI: s_endpgm
+; GCN-LABEL: @misaligned_read2_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15
+; GCN: s_endpgm
 define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i
@@ -382,9 +437,12 @@ define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, doubl
 
 @foo = addrspace(3) global [4 x i32] undef, align 4
 
-; SI-LABEL: @load_constant_adjacent_offsets
-; SI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1
+; GCN-LABEL: @load_constant_adjacent_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1
 define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) {
   %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
   %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4
@@ -393,9 +451,12 @@ define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out
   ret void
 }
 
-; SI-LABEL: @load_constant_disjoint_offsets
-; SI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:2
+; GCN-LABEL: @load_constant_disjoint_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:2
 define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) {
   %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
   %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4
@@ -406,10 +467,13 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out
 
 @bar = addrspace(3) global [4 x i64] undef, align 4
 
-; SI-LABEL: @load_misaligned64_constant_offsets
-; SI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset0:2 offset1:3
+; GCN-LABEL: @load_misaligned64_constant_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset0:2 offset1:3
 define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) {
   %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
   %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
@@ -420,12 +484,15 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)*
 
 @bar.large = addrspace(3) global [4096 x i64] undef, align 4
 
-; SI-LABEL: @load_misaligned64_constant_large_offsets
-; SI-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}}
-; SI-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000
-; SI-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE0]] offset1:1
-; SI-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE1]] offset1:1
-; SI: s_endpgm
+; GCN-LABEL: @load_misaligned64_constant_large_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}}
+; GCN-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000
+; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE0]] offset1:1
+; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE1]] offset1:1
+; GCN: s_endpgm
 define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) {
   %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4
   %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4
@@ -437,6 +504,10 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspac
 @sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4
 @sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4
 
+; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1
   %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1
@@ -481,20 +552,29 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %
   ret void
 }
 
+; GCN-LABEL: {{^}}misaligned_read2_v2i32:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 {
   %load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4
   store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8
   ret void
 }
 
+; GCN-LABEL: {{^}}misaligned_read2_i64:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 {
   %load = load i64, i64 addrspace(3)* %in, align 4
   store i64 %load, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; SI-LABEL: ds_read_diff_base_interleaving
-; SI-NOT: ds_read_b32
+; GCN-LABEL: ds_read_diff_base_interleaving
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read_b32
 define amdgpu_kernel void @ds_read_diff_base_interleaving(
   float addrspace(1)* nocapture %arg,
   [4 x [4 x float]] addrspace(3)* %arg1,
@@ -533,21 +613,13 @@ bb:
   ret void
 }
 
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workgroup.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workgroup.id.y() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() #1
 
-; Function Attrs: convergent nounwind
 declare void @llvm.amdgcn.s.barrier() #2
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind readnone }
+attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { convergent nounwind }
diff --git a/test/CodeGen/AMDGPU/ds_read2st64.ll b/test/CodeGen/AMDGPU/ds_read2st64.ll
index b1fba8c240d7c..c09dca4635f91 100644
--- a/test/CodeGen/AMDGPU/ds_read2st64.ll
+++ b/test/CodeGen/AMDGPU/ds_read2st64.ll
@@ -1,15 +1,19 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.f64 = addrspace(3) global [512 x double] undef, align 8
 
 
-; SI-LABEL: @simple_read2st64_f32_0_1
-; SI: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f32_0_1
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
+; CI: buffer_store_dword [[RESULT]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f32_0_1(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -23,12 +27,15 @@ define amdgpu_kernel void @simple_read2st64_f32_0_1(float addrspace(1)* %out) #0
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f32_1_2
-; SI: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:2
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f32_1_2
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:2
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
+; CI: buffer_store_dword [[RESULT]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f32_1_2(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -43,12 +50,15 @@ define amdgpu_kernel void @simple_read2st64_f32_1_2(float addrspace(1)* %out, fl
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f32_max_offset
-; SI: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:255
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
-; SI: buffer_store_dword [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f32_max_offset
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:255
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]]
+; CI: buffer_store_dword [[RESULT]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f32_max_offset(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -63,12 +73,15 @@ define amdgpu_kernel void @simple_read2st64_f32_max_offset(float addrspace(1)* %
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f32_over_max_offset
-; SI-NOT: ds_read2st64_b32
-; SI-DAG: v_add_i32_e32 [[BIGADD:v[0-9]+]], vcc, 0x10000, {{v[0-9]+}}
-; SI-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:256
-; SI-DAG: ds_read_b32 {{v[0-9]+}}, [[BIGADD]]{{$}}
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f32_over_max_offset
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st64_b32
+; GCN-DAG: v_add_{{i|u}}32_e32 [[BIGADD:v[0-9]+]], {{(vcc, )?}}0x10000, {{v[0-9]+}}
+; GCN-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:256
+; GCN-DAG: ds_read_b32 {{v[0-9]+}}, [[BIGADD]]{{$}}
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2st64_f32_over_max_offset(float addrspace(1)* %out, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -83,9 +96,12 @@ define amdgpu_kernel void @simple_read2st64_f32_over_max_offset(float addrspace(
   ret void
 }
 
-; SI-LABEL: @odd_invalid_read2st64_f32_0
-; SI-NOT: ds_read2st64_b32
-; SI: s_endpgm
+; GCN-LABEL: @odd_invalid_read2st64_f32_0
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st64_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @odd_invalid_read2st64_f32_0(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i
@@ -99,9 +115,12 @@ define amdgpu_kernel void @odd_invalid_read2st64_f32_0(float addrspace(1)* %out)
   ret void
 }
 
-; SI-LABEL: @odd_invalid_read2st64_f32_1
-; SI-NOT: ds_read2st64_b32
-; SI: s_endpgm
+; GCN-LABEL: @odd_invalid_read2st64_f32_1
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st64_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @odd_invalid_read2st64_f32_1(float addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -116,12 +135,15 @@ define amdgpu_kernel void @odd_invalid_read2st64_f32_1(float addrspace(1)* %out)
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f64_0_1
-; SI: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f64_0_1
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
+; CI: buffer_store_dwordx2 [[RESULT]]
+; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f64_0_1(double addrspace(1)* %out) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i
@@ -135,12 +157,16 @@ define amdgpu_kernel void @simple_read2st64_f64_0_1(double addrspace(1)* %out) #
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f64_1_2
-; SI: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:2
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f64_1_2
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:2
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
+
+; CI: buffer_store_dwordx2 [[RESULT]]
+; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f64_1_2(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -157,10 +183,13 @@ define amdgpu_kernel void @simple_read2st64_f64_1_2(double addrspace(1)* %out, d
 
 ; Alignment only
 
-; SI-LABEL: @misaligned_read2st64_f64
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1
-; SI: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:128 offset1:129
-; SI: s_endpgm
+; GCN-LABEL: @misaligned_read2st64_f64
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1
+; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:128 offset1:129
+; GCN: s_endpgm
 define amdgpu_kernel void @misaligned_read2st64_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i
@@ -175,12 +204,16 @@ define amdgpu_kernel void @misaligned_read2st64_f64(double addrspace(1)* %out, d
 }
 
 ; The maximum is not the usual 0xff because 0xff * 8 * 64 > 0xffff
-; SI-LABEL: @simple_read2st64_f64_max_offset
-; SI: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:4 offset1:127
-; SI: s_waitcnt lgkmcnt(0)
-; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
-; SI: buffer_store_dwordx2 [[RESULT]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f64_max_offset
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:4 offset1:127
+; GCN: s_waitcnt lgkmcnt(0)
+; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}
+
+; CI: buffer_store_dwordx2 [[RESULT]]
+; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @simple_read2st64_f64_max_offset(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 256
@@ -195,12 +228,15 @@ define amdgpu_kernel void @simple_read2st64_f64_max_offset(double addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_read2st64_f64_over_max_offset
-; SI-NOT: ds_read2st64_b64
-; SI-DAG: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset:512
-; SI-DAG: v_add_i32_e32 [[BIGADD:v[0-9]+]], vcc, 0x10000, {{v[0-9]+}}
-; SI: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, [[BIGADD]]
-; SI: s_endpgm
+; GCN-LABEL: @simple_read2st64_f64_over_max_offset
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st64_b64
+; GCN-DAG: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset:512
+; GCN-DAG: v_add_{{i|u}}32_e32 [[BIGADD:v[0-9]+]], {{(vcc, )?}}0x10000, {{v[0-9]+}}
+; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, [[BIGADD]]
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_read2st64_f64_over_max_offset(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -215,9 +251,12 @@ define amdgpu_kernel void @simple_read2st64_f64_over_max_offset(double addrspace
   ret void
 }
 
-; SI-LABEL: @invalid_read2st64_f64_odd_offset
-; SI-NOT: ds_read2st64_b64
-; SI: s_endpgm
+; GCN-LABEL: @invalid_read2st64_f64_odd_offset
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st64_b64
+; GCN: s_endpgm
 define amdgpu_kernel void @invalid_read2st64_f64_odd_offset(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %add.x.0 = add nsw i32 %x.i, 64
@@ -235,10 +274,13 @@ define amdgpu_kernel void @invalid_read2st64_f64_odd_offset(double addrspace(1)*
 ; The stride of 8 elements is 8 * 8 bytes. We need to make sure the
 ; stride in elements, not bytes, is a multiple of 64.
 
-; SI-LABEL: @byte_size_only_divisible_64_read2_f64
-; SI-NOT: ds_read2st_b64
-; SI: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @byte_size_only_divisible_64_read2_f64
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_read2st_b64
+; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @byte_size_only_divisible_64_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i
@@ -252,10 +294,7 @@ define amdgpu_kernel void @byte_size_only_divisible_64_read2_f64(double addrspac
   ret void
 }
 
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() #1
 
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/ds_write2.ll b/test/CodeGen/AMDGPU/ds_write2.ll
index 0f49919a1d109..7cb070c12b65d 100644
--- a/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/test/CodeGen/AMDGPU/ds_write2.ll
@@ -1,14 +1,17 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9 %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 @lds.f64 = addrspace(3) global [512 x double] undef, align 8
 
+; GCN-LABEL: {{^}}simple_write2_one_val_f32:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
 
-; SI-LABEL: @simple_write2_one_val_f32
-; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]]
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:8
-; SI: s_endpgm
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[VAL:v[0-9]+]]
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -21,12 +24,19 @@ define amdgpu_kernel void @simple_write2_one_val_f32(float addrspace(1)* %C, flo
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f32
-; SI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}simple_write2_two_val_f32:
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -41,11 +51,14 @@ define amdgpu_kernel void @simple_write2_two_val_f32(float addrspace(1)* %C, flo
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f32_volatile_0
-; SI-NOT: ds_write2_b32
-; SI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
-; SI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_f32_volatile_0
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_write2_b32
+; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
+; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
@@ -60,11 +73,14 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(float addrspace(
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f32_volatile_1
-; SI-NOT: ds_write2_b32
-; SI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
-; SI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_f32_volatile_1
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_write2_b32
+; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}}
+; GCN: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:32
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
@@ -80,12 +96,20 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(float addrspace(
 }
 
 ; 2 data subregisters from different super registers.
-; SI-LABEL: @simple_write2_two_val_subreg2_mixed_f32
-; SI: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}}
-; SI: buffer_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}}
-; SI: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}simple_write2_two_val_subreg2_mixed_f32:
+; GFX9-NOT: m0
+
+; CI: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}}
+; CI: buffer_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}}
+; CI-DAG: s_mov_b32 m0
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+
+; GFX9: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:{{[0-9]+\]}}
+; GFX9: global_load_dwordx2 v{{\[[0-9]+}}:[[VAL1:[0-9]+]]{{\]}}
+
+; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i
@@ -102,11 +126,14 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(float addrspa
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_subreg2_f32
-; SI-DAG: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_subreg2_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)* %C, <2 x float> addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %x.i
@@ -121,11 +148,14 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_subreg4_f32
-; SI-DAG: buffer_load_dwordx4 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_subreg4_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: {{buffer|global}}_load_dwordx4 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)* %C, <4 x float> addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 %x.i
@@ -140,12 +170,19 @@ define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_max_offset_f32
-; SI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_max_offset_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -160,10 +197,13 @@ define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(float addrspace(
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_too_far_f32
-; SI: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
-; SI: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_too_far_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
@@ -178,10 +218,13 @@ define amdgpu_kernel void @simple_write2_two_val_too_far_f32(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f32_x2
-; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8
-; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_f32_x2
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8
+; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x
@@ -208,10 +251,13 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2(float addrspace(1)* %C,
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f32_x2_nonzero_base
-; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8
-; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_f32_x2_nonzero_base
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8
+; GCN: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %tid.x
@@ -238,11 +284,14 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(float addrs
   ret void
 }
 
-; SI-LABEL: @write2_ptr_subreg_arg_two_val_f32
-; SI-NOT: ds_write2_b32
-; SI: ds_write_b32
-; SI: ds_write_b32
-; SI: s_endpgm
+; GCN-LABEL: @write2_ptr_subreg_arg_two_val_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_write2_b32
+; GCN: ds_write_b32
+; GCN: ds_write_b32
+; GCN: s_endpgm
 define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1, <2 x float addrspace(3)*> %lds.ptr) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in0.gep = getelementptr float, float addrspace(1)* %in0, i32 %x.i
@@ -265,11 +314,14 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2_one_val_f64
-; SI-DAG: buffer_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]],
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
-; SI: ds_write2_b64 [[VPTR]], [[VAL]], [[VAL]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_one_val_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: {{buffer|global}}_load_dwordx2 [[VAL:v\[[0-9]+:[0-9]+\]]],
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
+; GCN: ds_write2_b64 [[VPTR]], [[VAL]], [[VAL]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i
@@ -282,12 +334,15 @@ define amdgpu_kernel void @simple_write2_one_val_f64(double addrspace(1)* %C, do
   ret void
 }
 
-; SI-LABEL: @misaligned_simple_write2_one_val_f64
-; SI-DAG: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
-; SI: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:1
-; SI: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset0:14 offset1:15
-; SI: s_endpgm
+; GCN-LABEL: @misaligned_simple_write2_one_val_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: {{buffer|global}}_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
+; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset1:1
+; GCN: ds_write2_b32 [[VPTR]], v[[VAL0]], v[[VAL1]] offset0:14 offset1:15
+; GCN: s_endpgm
 define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i
@@ -300,12 +355,20 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(double addrspace
   ret void
 }
 
-; SI-LABEL: @simple_write2_two_val_f64
-; SI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
-; SI: ds_write2_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2_two_val_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
+
+; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off offset:8
+
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, v{{[0-9]+}}
+; GCN: ds_write2_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, double addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr double, double addrspace(1)* %in, i32 %x.i
@@ -322,19 +385,25 @@ define amdgpu_kernel void @simple_write2_two_val_f64(double addrspace(1)* %C, do
 
 @foo = addrspace(3) global [4 x i32] undef, align 4
 
-; SI-LABEL: @store_constant_adjacent_offsets
-; SI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
+; GCN-LABEL: @store_constant_adjacent_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
 define amdgpu_kernel void @store_constant_adjacent_offsets() {
   store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
   store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4
   ret void
 }
 
-; SI-LABEL: @store_constant_disjoint_offsets
-; SI-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b{{$}}
-; SI-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI: ds_write2_b32 [[ZERO]], [[VAL]], [[VAL]] offset1:2
+; GCN-LABEL: @store_constant_disjoint_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x7b{{$}}
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN: ds_write2_b32 [[ZERO]], [[VAL]], [[VAL]] offset1:2
 define amdgpu_kernel void @store_constant_disjoint_offsets() {
   store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4
   store i32 123, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4
@@ -343,11 +412,14 @@ define amdgpu_kernel void @store_constant_disjoint_offsets() {
 
 @bar = addrspace(3) global [4 x i64] undef, align 4
 
-; SI-LABEL: @store_misaligned64_constant_offsets
-; SI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
-; SI-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; SI-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3
-; SI: s_endpgm
+; GCN-LABEL: @store_misaligned64_constant_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}}
+; GCN-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
+; GCN-DAG: ds_write2_b32 [[ZERO]], v{{[0-9]+}}, v{{[0-9]+}} offset0:2 offset1:3
+; GCN: s_endpgm
 define amdgpu_kernel void @store_misaligned64_constant_offsets() {
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4
@@ -356,12 +428,15 @@ define amdgpu_kernel void @store_misaligned64_constant_offsets() {
 
 @bar.large = addrspace(3) global [4096 x i64] undef, align 4
 
-; SI-LABEL: @store_misaligned64_constant_large_offsets
-; SI-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}}
-; SI-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000{{$}}
-; SI-DAG: ds_write2_b32 [[BASE0]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; SI-DAG: ds_write2_b32 [[BASE1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
-; SI: s_endpgm
+; GCN-LABEL: @store_misaligned64_constant_large_offsets
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}}
+; GCN-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000{{$}}
+; GCN-DAG: ds_write2_b32 [[BASE0]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
+; GCN-DAG: ds_write2_b32 [[BASE1]], v{{[0-9]+}}, v{{[0-9]+}} offset1:1
+; GCN: s_endpgm
 define amdgpu_kernel void @store_misaligned64_constant_large_offsets() {
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4
   store i64 123, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4
@@ -406,10 +481,12 @@ define amdgpu_kernel void @write2_sgemm_sequence(float addrspace(1)* %C, i32 %ld
   ret void
 }
 
-; CI-LABEL: {{^}}simple_write2_v4f32_superreg_align4:
-; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:3 offset1:2{{$}}
-; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:1{{$}}
-; CI: s_endpgm
+; GCN-LABEL: {{^}}simple_write2_v4f32_superreg_align4:
+; CI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:2 offset1:3{{$}}
+; GCN: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset1:1{{$}}
 define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrspace(3)* %out, <4 x float> addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %in
@@ -419,18 +496,11 @@ define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(<4 x float> addrs
   ret void
 }
 
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workgroup.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workgroup.id.y() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() #1
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind readnone }
+attributes #1 = { nounwind readnone speculatable }
 attributes #2 = { convergent nounwind }
diff --git a/test/CodeGen/AMDGPU/ds_write2st64.ll b/test/CodeGen/AMDGPU/ds_write2st64.ll
index a395af34b67b9..54f2500afab4f 100644
--- a/test/CodeGen/AMDGPU/ds_write2st64.ll
+++ b/test/CodeGen/AMDGPU/ds_write2st64.ll
@@ -1,12 +1,16 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 @lds = addrspace(3) global [512 x float] undef, align 4
 
-; SI-LABEL: @simple_write2st64_one_val_f32_0_1
-; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]]
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2st64_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:1
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2st64_one_val_f32_0_1
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0n
+
+; GCN-DAG: {{buffer|global}}_load_dword [[VAL:v[0-9]+]]
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2st64_b32 [[VPTR]], [[VAL]], [[VAL]] offset1:1
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2st64_one_val_f32_0_1(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -19,12 +23,20 @@ define amdgpu_kernel void @simple_write2st64_one_val_f32_0_1(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2st64_two_val_f32_2_5
-; SI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset0:2 offset1:5
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2st64_two_val_f32_2_5
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4
+
+
+; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset0:2 offset1:5
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2st64_two_val_f32_2_5(float addrspace(1)* %C, float addrspace(1)* %in) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -40,12 +52,20 @@ define amdgpu_kernel void @simple_write2st64_two_val_f32_2_5(float addrspace(1)*
   ret void
 }
 
-; SI-LABEL: @simple_write2st64_two_val_max_offset_f32
-; SI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
-; SI-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 2, v{{[0-9]+}}
-; SI: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2st64_two_val_max_offset_f32
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
+
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, off offset:4
+
+; GCN-DAG: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v{{[0-9]+}}
+; GCN: v_add_{{i|u}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}s{{[0-9]+}}, [[SHL]]
+; GCN: ds_write2st64_b32 [[VPTR]], [[VAL0]], [[VAL1]] offset1:255
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f32(float addrspace(1)* %C, float addrspace(1)* %in, float addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr float, float addrspace(1)* %in, i32 %x.i
@@ -60,12 +80,20 @@ define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f32(float addrsp
   ret void
 }
 
-; SI-LABEL: @simple_write2st64_two_val_max_offset_f64
-; SI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
-; SI-DAG: v_add_i32_e32 [[VPTR:v[0-9]+]],
-; SI: ds_write2st64_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset0:4 offset1:127
-; SI: s_endpgm
+; GCN-LABEL: @simple_write2st64_two_val_max_offset_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; CI-DAG: buffer_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+; CI-DAG: buffer_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
+
+; GFX9-DAG: global_load_dwordx2 [[VAL0:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off{{$}}
+; GFX9-DAG: global_load_dwordx2 [[VAL1:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, off offset:8
+
+; GCN-DAG: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 3, v{{[0-9]+}}
+; GCN: v_add_{{i|u}}32_e32 [[VPTR:v[0-9]+]], {{(vcc, )?}}s{{[0-9]+}}, [[SHL]]
+; GCN: ds_write2st64_b64 [[VPTR]], [[VAL0]], [[VAL1]] offset0:4 offset1:127
+; GCN: s_endpgm
 define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep.0 = getelementptr double, double addrspace(1)* %in, i32 %x.i
@@ -81,10 +109,13 @@ define amdgpu_kernel void @simple_write2st64_two_val_max_offset_f64(double addrs
   ret void
 }
 
-; SI-LABEL: @byte_size_only_divisible_64_write2st64_f64
-; SI-NOT: ds_write2st64_b64
-; SI: ds_write2_b64 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:8
-; SI: s_endpgm
+; GCN-LABEL: @byte_size_only_divisible_64_write2st64_f64
+; CI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: ds_write2st64_b64
+; GCN: ds_write2_b64 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:8
+; GCN: s_endpgm
 define amdgpu_kernel void @byte_size_only_divisible_64_write2st64_f64(double addrspace(1)* %C, double addrspace(1)* %in, double addrspace(3)* %lds) #0 {
   %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %in.gep = getelementptr double, double addrspace(1)* %in, i32 %x.i
@@ -97,10 +128,7 @@ define amdgpu_kernel void @byte_size_only_divisible_64_write2st64_f64(double add
   ret void
 }
 
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.x() #1
-
-; Function Attrs: nounwind readnone
 declare i32 @llvm.amdgcn.workitem.id.y() #1
 
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index b1107ea7fbcbf..d0efd07497d90 100644
--- a/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -1,11 +1,12 @@
-; RUN: not llc -march=amdgcn -mcpu=tahiti -mattr=+promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
-; RUN: not llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
-; RUN: not llc -march=r600 -mcpu=cypress < %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tahiti -mattr=+promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s 2>&1 | FileCheck %s
+; RUN: not llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s 2>&1 | FileCheck %s
+target datalayout = "A5"
 
 ; CHECK: in function test_dynamic_stackalloc{{.*}}: unsupported dynamic alloca
 
 define amdgpu_kernel void @test_dynamic_stackalloc(i32 addrspace(1)* %out, i32 %n) {
-  %alloca = alloca i32, i32 %n
-  store volatile i32 0, i32* %alloca
+  %alloca = alloca i32, i32 %n, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/early-if-convert.ll b/test/CodeGen/AMDGPU/early-if-convert.ll
index 792f0b1eaef46..d129ca5c140af 100644
--- a/test/CodeGen/AMDGPU/early-if-convert.ll
+++ b/test/CodeGen/AMDGPU/early-if-convert.ll
@@ -382,7 +382,7 @@ done:
 }
 
 ; GCN-LABEL: {{^}}ifcvt_undef_scc:
-; GCN: {{^}}; BB#0:
+; GCN: {{^}}; %bb.0:
 ; GCN-NEXT: s_load_dwordx2
 ; GCN-NEXT: s_cselect_b32 s{{[0-9]+}}, 1, 0
 define amdgpu_kernel void @ifcvt_undef_scc(i32 %cond, i32 addrspace(1)* %out) {
diff --git a/test/CodeGen/AMDGPU/early-inline.ll b/test/CodeGen/AMDGPU/early-inline.ll
index c871d54bec7ed..a4f970ee238a8 100644
--- a/test/CodeGen/AMDGPU/early-inline.ll
+++ b/test/CodeGen/AMDGPU/early-inline.ll
@@ -1,6 +1,5 @@
 ; RUN: opt -mtriple=amdgcn-- -O1 -S -inline-threshold=1 -amdgpu-early-inline-all %s | FileCheck %s
 
-; CHECK: @c_alias
 @c_alias = alias i32 (i32), i32 (i32)* @callee
 
 define i32 @callee(i32 %x) {
@@ -17,6 +16,7 @@ entry:
 ; CHECK: mul i32
 ; CHECK-NOT: call i32
 
+; CHECK: define i32 @c_alias
 define amdgpu_kernel void @caller(i32 %x) {
 entry:
   %res = call i32 @callee(i32 %x)
diff --git a/test/CodeGen/AMDGPU/elf-header.ll b/test/CodeGen/AMDGPU/elf-header.ll
new file mode 100644
index 0000000000000..192eb780f70e8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/elf-header.ll
@@ -0,0 +1,49 @@
+; RUN: llc -march=r600 -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=R600 --check-prefix=R600-OSABI-NONE %s
+; RUN: llc -mtriple=r600-- -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=R600 --check-prefix=R600-OSABI-NONE %s
+; RUN: llc -mtriple=r600-amd- -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=R600 --check-prefix=R600-OSABI-NONE %s
+; RUN: llc -mtriple=r600-amd-unknown -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=R600 --check-prefix=R600-OSABI-NONE %s
+; RUN: llc -mtriple=r600-unknown-unknown -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=R600 --check-prefix=R600-OSABI-NONE %s
+
+; RUN: llc -march=amdgcn -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-NONE %s
+; RUN: llc -mtriple=amdgcn-- -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-NONE %s
+; RUN: llc -mtriple=amdgcn-amd- -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-NONE %s
+; RUN: llc -mtriple=amdgcn-amd-unknown -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-NONE %s
+; RUN: llc -mtriple=amdgcn-unknown-unknown -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-NONE %s
+
+; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-HSA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-HSA %s
+; RUN: llc -mtriple=amdgcn-unknown-amdhsa -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-HSA %s
+
+; RUN: llc -mtriple=amdgcn--amdpal -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-PAL %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-PAL %s
+; RUN: llc -mtriple=amdgcn-unknown-amdpal -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-PAL %s
+
+; RUN: llc -mtriple=amdgcn--mesa3d -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-MESA3D %s
+; RUN: llc -mtriple=amdgcn-amd-mesa3d -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-MESA3D %s
+; RUN: llc -mtriple=amdgcn-unknown-mesa3d -filetype=obj < %s | llvm-readobj -file-headers - | FileCheck --check-prefix=GCN --check-prefix=GCN-OSABI-MESA3D %s
+
+; R600: Format: ELF32-amdgpu
+; R600: Arch: r600
+; R600: AddressSize: 32bit
+; GCN:  Format: ELF64-amdgpu
+; GCN:  Arch: amdgcn
+; GCN:  AddressSize: 64bit
+
+; R600-OSABI-NONE:  OS/ABI: SystemV (0x0)
+; GCN-OSABI-NONE:   OS/ABI: SystemV (0x0)
+; GCN-OSABI-HSA:    OS/ABI: AMDGPU_HSA (0x40)
+; GCN-OSABI-PAL:    OS/ABI: AMDGPU_PAL (0x41)
+; GCN-OSABI-MESA3D: OS/ABI: AMDGPU_MESA3D (0x42)
+
+; R600: Machine: EM_AMDGPU (0xE0)
+; R600: Flags [ (0x1)
+; R600:   EF_AMDGPU_ARCH_R600 (0x1)
+; R600: ]
+; GCN:  Machine: EM_AMDGPU (0xE0)
+; GCN:  Flags [ (0x2)
+; GCN:    EF_AMDGPU_ARCH_GCN (0x2)
+; GCN:  ]
+
+define amdgpu_kernel void @elf_header() {
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/elf-notes.ll b/test/CodeGen/AMDGPU/elf-notes.ll
new file mode 100644
index 0000000000000..cd3c91719592a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/elf-notes.ll
@@ -0,0 +1,85 @@
+; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=gfx800 -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=iceland -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-unknown -mcpu=gfx800 -mattr=+code-object-v3 -filetype=obj < %s | llvm-readobj -elf-output-style=GNU -notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-UNK-ELF --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=iceland -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -mattr=+code-object-v3 -filetype=obj < %s | llvm-readobj -elf-output-style=GNU -notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-HSA-ELF --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx800 -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=iceland -mattr=+code-object-v3 < %s | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL --check-prefix=GFX800 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx800 -mattr=+code-object-v3 -filetype=obj < %s | llvm-readobj -elf-output-style=GNU -notes  | FileCheck --check-prefix=GCN --check-prefix=OSABI-PAL-ELF --check-prefix=GFX800 %s
+; RUN: llc -march=r600 -mattr=+code-object-v3 < %s | FileCheck --check-prefix=R600 %s
+
+; OSABI-UNK-NOT: .hsa_code_object_version
+; OSABI-UNK-NOT: .hsa_code_object_isa
+; OSABI-UNK: .amd_amdgpu_isa "amdgcn-amd-unknown--gfx800"
+; OSABI-UNK-NOT: .amd_amdgpu_hsa_metadata
+; OSABI-UNK-NOT: .amd_amdgpu_pal_metadata
+
+; OSABI-UNK-ELF-NOT: Unknown note type
+; OSABI-UNK-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
+; OSABI-UNK-ELF: ISA Version:
+; OSABI-UNK-ELF: amdgcn-amd-unknown--gfx800
+; OSABI-UNK-ELF-NOT: Unknown note type
+; OSABI-UNK-ELF-NOT: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-UNK-ELF-NOT: Unknown note type
+; OSABI-UNK-ELF-NOT: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+; OSABI-UNK-ELF-NOT: Unknown note type
+
+; OSABI-HSA-NOT: .hsa_code_object_version
+; OSABI-HSA-NOT: .hsa_code_object_isa
+; OSABI-HSA: .amd_amdgpu_isa "amdgcn-amd-amdhsa--gfx800"
+; OSABI-HSA: .amd_amdgpu_hsa_metadata
+; OSABI-HSA-NOT: .amd_amdgpu_pal_metadata
+
+; OSABI-HSA-ELF-NOT: Unknown note type
+; OSABI-HSA-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
+; OSABI-HSA-ELF: ISA Version:
+; OSABI-HSA-ELF: amdgcn-amd-amdhsa--gfx800
+; OSABI-HSA-ELF-NOT: Unknown note type
+; OSABI-HSA-ELF: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-HSA-ELF: HSA Metadata:
+; OSABI-HSA-ELF: ---
+; OSABI-HSA-ELF: Version: [ 1, 0 ]
+; OSABI-HSA-ELF: Kernels:
+; OSABI-HSA-ELF:   - Name:       elf_notes
+; OSABI-HSA-ELF:     SymbolName: 'elf_notes@kd'
+; OSABI-HSA-ELF:     CodeProps:
+; OSABI-HSA-ELF:       KernargSegmentSize: 0
+; OSABI-HSA-ELF:       GroupSegmentFixedSize: 0
+; OSABI-HSA-ELF:       PrivateSegmentFixedSize: 0
+; OSABI-HSA-ELF:       KernargSegmentAlign: 4
+; OSABI-HSA-ELF:       WavefrontSize:   64
+; OSABI-HSA-ELF:       NumSGPRs:        96
+; OSABI-HSA-ELF: ...
+; OSABI-HSA-ELF-NOT: Unknown note type
+; OSABI-HSA-ELF-NOT: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+; OSABI-HSA-ELF-NOT: Unknown note type
+
+; OSABI-PAL-NOT: .hsa_code_object_version
+; OSABI-PAL-NOT: .hsa_code_object_isa
+; OSABI-PAL: .amd_amdgpu_isa "amdgcn-amd-amdpal--gfx800"
+; OSABI-PAL-NOT: .amd_amdgpu_hsa_metadata
+; OSABI-PAL: .amd_amdgpu_pal_metadata
+
+; OSABI-PAL-ELF-NOT: Unknown note type
+; OSABI-PAL-ELF: NT_AMD_AMDGPU_ISA (ISA Version)
+; OSABI-PAL-ELF: ISA Version:
+; OSABI-PAL-ELF: amdgcn-amd-amdpal--gfx800
+; OSABI-PAL-ELF-NOT: Unknown note type
+; OSABI-PAL-ELF-NOT: NT_AMD_AMDGPU_HSA_METADATA (HSA Metadata)
+; OSABI-PAL-ELF-NOT: Unknown note type
+; OSABI-PAL-ELF: NT_AMD_AMDGPU_PAL_METADATA (PAL Metadata)
+; OSABI-PAL-ELF: PAL Metadata:
+; TODO: Following check line fails on mips:
+; OSABI-PAL-ELF-XXX: 0x2e12,0xac02c0,0x2e13,0x80,0x1000001b,0x1,0x10000022,0x60,0x1000003e,0x0
+; OSABI-PAL-ELF-NOT: Unknown note type
+
+; R600-NOT: .hsa_code_object_version
+; R600-NOT: .hsa_code_object_isa
+; R600-NOT: .amd_amdgpu_isa
+; R600-NOT: .amd_amdgpu_hsa_metadata
+; R600-NOT: .amd_amdgpu_pal_metadatas
+
+define amdgpu_kernel void @elf_notes() {
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/elf.ll b/test/CodeGen/AMDGPU/elf.ll
index b22f8608d7e33..de8c010f204aa 100644
--- a/test/CodeGen/AMDGPU/elf.ll
+++ b/test/CodeGen/AMDGPU/elf.ll
@@ -8,8 +8,8 @@
 ; Test that we don't try to produce a COFF file on windows
 ; RUN: llc < %s -mtriple=amdgcn-pc-mingw -verify-machineinstrs -filetype=obj | llvm-readobj -s -symbols -file-headers - | FileCheck --check-prefix=ELF %s
 
-; ELF: Format: ELF64
-; ELF: OS/ABI: AMDGPU_HSA (0x40)
+; ELF: Format: ELF64-amdgpu
+; ELF: OS/ABI: SystemV (0x0)
 ; ELF: Machine: EM_AMDGPU (0xE0)
 ; ELF: Name: .AMDGPU.config
 ; ELF: Type: SHT_PROGBITS
diff --git a/test/CodeGen/AMDGPU/elf.r600.ll b/test/CodeGen/AMDGPU/elf.r600.ll
index 93c5e55750336..1ca1524cbaa1c 100644
--- a/test/CodeGen/AMDGPU/elf.r600.ll
+++ b/test/CodeGen/AMDGPU/elf.r600.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -march=r600 -mcpu=redwood -filetype=obj | llvm-readobj -s - | FileCheck --check-prefix=ELF %s
 ; RUN: llc < %s -march=r600 -mcpu=redwood -o - | FileCheck --check-prefix=CONFIG %s
 
-; ELF: Format: ELF32
+; ELF: Format: ELF32-amdgpu
 ; ELF: Name: .AMDGPU.config
 
 ; CONFIG: .section .AMDGPU.config
diff --git a/test/CodeGen/AMDGPU/else.ll b/test/CodeGen/AMDGPU/else.ll
index 22338e4f50e56..c73ea936e8be4 100644
--- a/test/CodeGen/AMDGPU/else.ll
+++ b/test/CodeGen/AMDGPU/else.ll
@@ -25,7 +25,7 @@ end:
 }
 
 ; CHECK-LABEL: {{^}}else_execfix_leave_wqm:
-; CHECK: ; BB#0:
+; CHECK: ; %bb.0:
 ; CHECK-NEXT: s_mov_b64 [[INIT_EXEC:s\[[0-9]+:[0-9]+\]]], exec
 ; CHECK: ; %Flow
 ; CHECK-NEXT: s_or_saveexec_b64 [[DST:s\[[0-9]+:[0-9]+\]]],
diff --git a/test/CodeGen/AMDGPU/empty-function.ll b/test/CodeGen/AMDGPU/empty-function.ll
index 1231cb4d1de2b..7058b4149b407 100644
--- a/test/CodeGen/AMDGPU/empty-function.ll
+++ b/test/CodeGen/AMDGPU/empty-function.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 ; Make sure we don't assert on empty functions
diff --git a/test/CodeGen/AMDGPU/endpgm-dce.mir b/test/CodeGen/AMDGPU/endpgm-dce.mir
new file mode 100644
index 0000000000000..9833cc10d40af
--- /dev/null
+++ b/test/CodeGen/AMDGPU/endpgm-dce.mir
@@ -0,0 +1,297 @@
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-optimize-exec-masking-pre-ra %s -o - | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: kill_all
+# GCN:      bb.0:
+# GCN-NEXT: S_ENDPGM
+name: kill_all
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_32 }
+  - { id: 4, class: sgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %0 = IMPLICIT_DEF
+    %3 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    %1 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+    %2 = V_ADD_F32_e64 0, killed %1, 0, 1, 0, 0, implicit %exec
+    %4 = S_ADD_U32 %3, 1, implicit-def %scc
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: load_without_memoperand
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: dead %1:vgpr_32 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr
+# GCN-NEXT: S_ENDPGM
+name: load_without_memoperand
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_32 }
+  - { id: 4, class: sgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %0 = IMPLICIT_DEF
+    %3 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    %1 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr
+    %2 = V_ADD_F32_e64 0, killed %1, 0, 1, 0, 0, implicit %exec
+    %4 = S_ADD_U32 %3, 1, implicit-def %scc
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: load_volatile
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: dead %1:vgpr_32 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile load 4)
+# GCN-NEXT: S_ENDPGM
+name: load_volatile
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sgpr_32 }
+  - { id: 4, class: sgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %0 = IMPLICIT_DEF
+    %3 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    %1 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile load 4)
+    %2 = V_ADD_F32_e64 0, killed %1, 0, 1, 0, 0, implicit %exec
+    %4 = S_ADD_U32 %3, 1, implicit-def %scc
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: store
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: FLAT_STORE_DWORD %0, %1, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4)
+# GCN-NEXT: S_ENDPGM
+name: store
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %0 = IMPLICIT_DEF
+    %1 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    FLAT_STORE_DWORD %0, %1, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4)
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: barrier
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: S_BARRIER
+# GCN-NEXT: S_ENDPGM
+name: barrier
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    S_BARRIER
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: call
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: %sgpr4_sgpr5 = S_SWAPPC_B64 %sgpr2_sgpr3
+# GCN-NEXT: S_ENDPGM
+name: call
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    %sgpr4_sgpr5 = S_SWAPPC_B64 %sgpr2_sgpr3
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: exp
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: EXP 32, undef %0:vgpr_32, undef %1:vgpr_32, %2, undef %3:vgpr_32, 0, 0, 15, implicit %exec
+# GCN-NEXT: S_ENDPGM
+name: exp
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vgpr_32 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %2 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    EXP 32, undef %0, undef %1, killed %2, undef %3, 0, 0, 15, implicit %exec
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: return_to_epilog
+# GCN:      %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT: SI_RETURN_TO_EPILOG killed %vgpr0
+name: return_to_epilog
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %vgpr0 = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    SI_RETURN_TO_EPILOG killed %vgpr0
+...
+---
+# GCN-LABEL: name: split_block
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.1
+# GCN-NOT:  S_OR_B64
+# GCN:      bb.1:
+# GCN-NEXT:   S_ENDPGM
+name: split_block
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vgpr_32 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: sgpr_32 }
+  - { id: 3, class: sgpr_32 }
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+
+  bb.1:
+    %0 = IMPLICIT_DEF
+    %2 = IMPLICIT_DEF
+    %1 = V_ADD_F32_e64 0, killed %0, 0, 1, 0, 0, implicit %exec
+    %3 = S_ADD_U32 %2, 1, implicit-def %scc
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: split_block_empty_block
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.1
+# GCN-NOT:  S_OR_B64
+# GCN:      bb.1:
+# GCN:      bb.2:
+# GCN-NEXT:   S_ENDPGM
+name: split_block_empty_block
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+
+  bb.1:
+
+  bb.2:
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: split_block_uncond_branch
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.1
+# GCN:        S_BRANCH %bb.1
+# GCN-NOT:  S_OR_B64
+# GCN:      bb.1:
+# GCN-NEXT:   S_ENDPGM
+name: split_block_uncond_branch
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    S_BRANCH %bb.1
+
+  bb.1:
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: split_block_cond_branch
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+# GCN:        %sgpr0_sgpr1 = S_OR_B64 %exec, %vcc, implicit-def %scc
+# GCN:        S_CBRANCH_VCCNZ %bb.2, implicit undef %vcc
+# GCN:      bb.1:
+# GCN:      bb.2:
+# GCN-NEXT:   S_ENDPGM
+name: split_block_cond_branch
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, %vcc, implicit-def %scc
+    S_CBRANCH_VCCNZ %bb.2, implicit undef %vcc
+
+  bb.1:
+
+  bb.2:
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: two_preds_both_dead
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.2
+# GCN-NOT:    S_OR
+# GCN:        S_BRANCH %bb.2
+# GCN:      bb.1:
+# GCN-NEXT:   successors: %bb.2
+# GCN-NOT:    S_AND
+# GCN:        S_BRANCH %bb.2
+# GCN:      bb.2:
+# GCN-NEXT:   S_ENDPGM
+name: two_preds_both_dead
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    S_BRANCH %bb.2
+
+  bb.1:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_AND_B64 %exec, killed %vcc, implicit-def %scc
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: two_preds_one_dead
+# GCN:      bb.0:
+# GCN-NEXT:   successors: %bb.2
+# GCN:        %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+# GCN-NEXT:   S_BARRIER
+# GCN-NEXT:   S_BRANCH %bb.2
+# GCN:      bb.1:
+# GCN-NEXT:   successors: %bb.2
+# GCN-NOT:    S_AND
+# GCN:        S_BRANCH %bb.2
+# GCN:      bb.2:
+# GCN-NEXT:   S_ENDPGM
+name: two_preds_one_dead
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    S_BARRIER
+    S_BRANCH %bb.2
+
+  bb.1:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_AND_B64 %exec, killed %vcc, implicit-def %scc
+    S_BRANCH %bb.2
+
+  bb.2:
+    S_ENDPGM
+...
diff --git a/test/CodeGen/AMDGPU/enqueue-kernel.ll b/test/CodeGen/AMDGPU/enqueue-kernel.ll
new file mode 100644
index 0000000000000..a54453541ded5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/enqueue-kernel.ll
@@ -0,0 +1,107 @@
+; RUN: opt -amdgpu-lower-enqueued-block -S < %s | FileCheck %s
+
+; CHECK: @__test_block_invoke_kernel_runtime_handle = external addrspace(1) externally_initialized constant i8 addrspace(1)*
+; CHECK: @__test_block_invoke_2_kernel_runtime_handle = external addrspace(1) externally_initialized constant i8 addrspace(1)*
+
+target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64"
+target triple = "amdgcn-amdhsa-amd-opencl"
+
+%struct.ndrange_t = type { i32 }
+%opencl.queue_t = type opaque
+
+; CHECK: define amdgpu_kernel void @non_caller(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr !kernel_arg_addr_space
+define amdgpu_kernel void @non_caller(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr
+  !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 {
+  ret void
+}
+
+; CHECK: define amdgpu_kernel void @caller_indirect(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr #[[AT_CALLER:[0-9]+]]
+define amdgpu_kernel void @caller_indirect(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr
+  !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 {
+  call void @caller(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d)
+  ret void
+}
+
+; CHECK: define amdgpu_kernel void @caller(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr #[[AT_CALLER]]
+define amdgpu_kernel void @caller(i8 addrspace(1)* %a, i8 %b, i64 addrspace(1)* %c, i64 %d) local_unnamed_addr
+  !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !5 !kernel_arg_type_qual !6 {
+entry:
+  %block = alloca <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, align 8
+  %tmp = alloca %struct.ndrange_t, align 4
+  %block2 = alloca <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, align 8
+  %tmp3 = alloca %struct.ndrange_t, align 4
+  %block.size = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block, i32 0, i32 0
+  store i32 25, i32* %block.size, align 8
+  %block.align = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block, i32 0, i32 1
+  store i32 8, i32* %block.align, align 4
+  %block.invoke = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block, i32 0, i32 2
+  store i8 addrspace(4)* addrspacecast (i8* bitcast (void (<{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>)* @__test_block_invoke_kernel to i8*) to i8 addrspace(4)*), i8 addrspace(4)** %block.invoke, align 8
+  %block.captured = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block, i32 0, i32 3
+  store i8 addrspace(1)* %a, i8 addrspace(1)** %block.captured, align 8
+  %block.captured1 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block, i32 0, i32 4
+  store i8 %b, i8* %block.captured1, align 8
+  %tmp1 = bitcast <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }>* %block to void ()*
+  %tmp2 = bitcast void ()* %tmp1 to i8*
+  %tmp4 = addrspacecast i8* %tmp2 to i8 addrspace(4)*
+  %tmp5 = call i32 @__enqueue_kernel_basic(%opencl.queue_t addrspace(1)* undef, i32 0, %struct.ndrange_t* byval nonnull %tmp, i8 addrspace(4)* nonnull %tmp4) #2
+  %block.size4 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 0
+  store i32 41, i32* %block.size4, align 8
+  %block.align5 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 1
+  store i32 8, i32* %block.align5, align 4
+  %block.invoke6 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 2
+  store i8 addrspace(4)* addrspacecast (i8* bitcast (void (<{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>)* @__test_block_invoke_2_kernel to i8*) to i8 addrspace(4)*), i8 addrspace(4)** %block.invoke6, align 8
+  %block.captured7 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 3
+  store i8 addrspace(1)* %a, i8 addrspace(1)** %block.captured7, align 8
+  %block.captured8 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 6
+  store i8 %b, i8* %block.captured8, align 8
+  %block.captured9 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 4
+  store i64 addrspace(1)* %c, i64 addrspace(1)** %block.captured9, align 8
+  %block.captured10 = getelementptr inbounds <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>, <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2, i32 0, i32 5
+  store i64 %d, i64* %block.captured10, align 8
+  %tmp6 = bitcast <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }>* %block2 to void ()*
+  %tmp7 = bitcast void ()* %tmp6 to i8*
+  %tmp8 = addrspacecast i8* %tmp7 to i8 addrspace(4)*
+  %tmp9 = call i32 @__enqueue_kernel_basic(%opencl.queue_t addrspace(1)* undef, i32 0, %struct.ndrange_t* byval nonnull %tmp3, i8 addrspace(4)* nonnull %tmp8) #2
+  ret void
+}
+
+; CHECK: define amdgpu_kernel void @__test_block_invoke_kernel({{.*}}) #[[AT1:[0-9]+]]
+define internal amdgpu_kernel void @__test_block_invoke_kernel(<{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }> %arg) #0
+  !kernel_arg_addr_space !14 !kernel_arg_access_qual !15 !kernel_arg_type !16 !kernel_arg_base_type !16 !kernel_arg_type_qual !17 {
+entry:
+  %.fca.3.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }> %arg, 3
+  %.fca.4.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }> %arg, 4
+  store i8 %.fca.4.extract, i8 addrspace(1)* %.fca.3.extract, align 1
+  ret void
+}
+
+declare i32 @__enqueue_kernel_basic(%opencl.queue_t addrspace(1)*, i32, %struct.ndrange_t*, i8 addrspace(4)*) local_unnamed_addr
+
+; CHECK: define amdgpu_kernel void @__test_block_invoke_2_kernel({{.*}}) #[[AT2:[0-9]+]]
+define internal amdgpu_kernel void @__test_block_invoke_2_kernel(<{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*,
+  i64 addrspace(1)*, i64, i8 }> %arg) #0 !kernel_arg_addr_space !14 !kernel_arg_access_qual !15
+  !kernel_arg_type !16 !kernel_arg_base_type !16 !kernel_arg_type_qual !17 {
+entry:
+  %.fca.3.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }> %arg, 3
+  %.fca.4.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }> %arg, 4
+  %.fca.5.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }> %arg, 5
+  %.fca.6.extract = extractvalue <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i64 addrspace(1)*, i64, i8 }> %arg, 6
+  store i8 %.fca.6.extract, i8 addrspace(1)* %.fca.3.extract, align 1
+  store i64 %.fca.5.extract, i64 addrspace(1)* %.fca.4.extract, align 8
+  ret void
+}
+
+; CHECK: attributes #[[AT_CALLER]] = { "calls-enqueue-kernel" }
+; CHECK: attributes #[[AT1]] = {{.*}}"runtime-handle"="__test_block_invoke_kernel_runtime_handle"
+; CHECK: attributes #[[AT2]] = {{.*}}"runtime-handle"="__test_block_invoke_2_kernel_runtime_handle"
+
+attributes #0 = { "enqueued-block" }
+
+!3 = !{i32 1, i32 0, i32 1, i32 0}
+!4 = !{!"none", !"none", !"none", !"none"}
+!5 = !{!"char*", !"char", !"long*", !"long"}
+!6 = !{!"", !"", !"", !""}
+!14 = !{i32 0}
+!15 = !{!"none"}
+!16 = !{!"__block_literal"}
+!17 = !{!""}
diff --git a/test/CodeGen/AMDGPU/extload-align.ll b/test/CodeGen/AMDGPU/extload-align.ll
index 12cf27b918afc..097ea2f0bc1e8 100644
--- a/test/CodeGen/AMDGPU/extload-align.ll
+++ b/test/CodeGen/AMDGPU/extload-align.ll
@@ -1,4 +1,5 @@
-; RUN: llc -debug-only=machine-scheduler -march=amdgcn -verify-machineinstrs %s -o - 2>&1| FileCheck -check-prefix=SI-NOHSA -check-prefix=FUNC -check-prefix=DEBUG %s
+; RUN: llc -debug-only=machine-scheduler -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs %s -o - 2>&1| FileCheck -check-prefix=SI-NOHSA -check-prefix=FUNC -check-prefix=DEBUG %s
+target datalayout = "A5"
 ; REQUIRES: asserts
 
 ; Verify that the extload generated from %eval has the default
@@ -6,18 +7,18 @@
 ; size and not 4 corresponding to the sign-extended size (i32).
 
 ; DEBUG: {{^}}# Machine code for function extload_align:
-; DEBUG: mem:LD2[<unknown>]{{[^(]}}
+; DEBUG: mem:LD2[<unknown>(addrspace=5)]{{[^(]}}
 ; DEBUG: {{^}}# End machine code for function extload_align.
 
-define amdgpu_kernel void @extload_align(i32* %out, i32 %index) #0 {
-  %v0 = alloca [4 x i16]
-  %a1 = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 0
-  %a2 = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 1
-  store i16 0, i16* %a1
-  store i16 1, i16* %a2
-  %a = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 %index
-  %val = load i16, i16* %a
+define amdgpu_kernel void @extload_align(i32 addrspace(5)* %out, i32 %index) #0 {
+  %v0 = alloca [4 x i16], addrspace(5)
+  %a1 = getelementptr inbounds [4 x i16], [4 x i16] addrspace(5)* %v0, i32 0, i32 0
+  %a2 = getelementptr inbounds [4 x i16], [4 x i16] addrspace(5)* %v0, i32 0, i32 1
+  store i16 0, i16 addrspace(5)* %a1
+  store i16 1, i16 addrspace(5)* %a2
+  %a = getelementptr inbounds [4 x i16], [4 x i16] addrspace(5)* %v0, i32 0, i32 %index
+  %val = load i16, i16 addrspace(5)* %a
   %eval = sext i16 %val to i32
-  store i32 %eval, i32* %out
+  store i32 %eval, i32 addrspace(5)* %out
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll b/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
index 1f567ae050817..1e19ddf2b0ac6 100644
--- a/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
+++ b/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}extract_vector_elt_v2f16:
 ; GCN: s_load_dword [[VEC:s[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll b/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
index db5bf0b4e8089..2f13f63fa8882 100644
--- a/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
+++ b/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}extract_vector_elt_v3f64_2:
 ; GCN: buffer_load_dwordx4
diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll b/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
index 9b117d48a9804..06a0e2c7b6537 100644
--- a/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
+++ b/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SICIVI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}extract_vector_elt_v2i16:
 ; GCN: s_load_dword [[VEC:s[0-9]+]]
@@ -36,7 +36,7 @@ define amdgpu_kernel void @extract_vector_elt_v2i16_dynamic_sgpr(i16 addrspace(1
 
 ; GCN-LABEL: {{^}}extract_vector_elt_v2i16_dynamic_vgpr:
 ; GCN-DAG: s_load_dword [[VEC:s[0-9]+]]
-; GCN-DAG: {{flat|buffer}}_load_dword [[IDX:v[0-9]+]]
+; GCN-DAG: {{flat|buffer|global}}_load_dword [[IDX:v[0-9]+]]
 ; GCN: v_lshlrev_b32_e32 [[IDX_SCALED:v[0-9]+]], 16, [[IDX]]
 
 ; SI: v_lshr_b32_e32 [[ELT:v[0-9]+]], [[VEC]], [[IDX_SCALED]]
@@ -92,19 +92,24 @@ define amdgpu_kernel void @extract_vector_elt_v4i16(i16 addrspace(1)* %out, <4 x
 }
 
 ; GCN-LABEL: {{^}}dynamic_extract_vector_elt_v3i16:
-; GCN: buffer_load_ushort
-; GCN: buffer_load_ushort
-; GCN: buffer_load_ushort
+; SICIVI: buffer_load_ushort
+; SICIVI: buffer_load_ushort
+; SICIVI: buffer_load_ushort
 
 ; SICIVI: buffer_store_short
 ; SICIVI: buffer_store_short
 ; SICIVI: buffer_store_short
 
+; SICIVI: buffer_load_ushort
+; SICIVI: buffer_store_short
+
+; GFX9: buffer_load_ushort
+; GFX9: global_load_short_d16_hi
+; GFX9: global_load_short_d16 v
 ; GFX9: buffer_store_dword
 ; GFX9: buffer_store_dword
-
-; GCN: buffer_load_ushort
-; GCN: buffer_store_short
+; GFX9: buffer_load_ushort
+; GFX9: buffer_store_short
 define amdgpu_kernel void @dynamic_extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> %foo, i32 %idx) #0 {
   %p0 = extractelement <3 x i16> %foo, i32 %idx
   %out1 = getelementptr i16, i16 addrspace(1)* %out, i32 1
diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll b/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
index a8d127879a32d..99019e2a83e61 100644
--- a/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
+++ b/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; How the replacement of i64 stores with v2i32 stores resulted in
 ; breaking other users of the bitcast if they already existed
diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll b/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
index b7d768fd55254..a5ff0932c6228 100644
--- a/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
+++ b/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}extract_vector_elt_v1i8:
 ; GCN: buffer_load_ubyte
diff --git a/test/CodeGen/AMDGPU/fabs.f16.ll b/test/CodeGen/AMDGPU/fabs.f16.ll
index d56d5ec1411a9..4bbaf0ea3b621 100644
--- a/test/CodeGen/AMDGPU/fabs.f16.ll
+++ b/test/CodeGen/AMDGPU/fabs.f16.ll
@@ -1,15 +1,15 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; DAGCombiner will transform:
 ; (fabs (f16 bitcast (i16 a))) => (f16 bitcast (and (i16 a), 0x7FFFFFFF))
 ; unless isFabsFree returns true
 
 ; GCN-LABEL: {{^}}s_fabs_free_f16:
-; GCN: flat_load_ushort [[VAL:v[0-9]+]],
+; GCN: {{flat|global}}_load_ushort [[VAL:v[0-9]+]],
 ; GCN: v_and_b32_e32 [[RESULT:v[0-9]+]], 0x7fff, [[VAL]]
-; GCN: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 
 define amdgpu_kernel void @s_fabs_free_f16(half addrspace(1)* %out, i16 %in) {
   %bc= bitcast i16 %in to half
@@ -20,7 +20,7 @@ define amdgpu_kernel void @s_fabs_free_f16(half addrspace(1)* %out, i16 %in) {
 
 ; GCN-LABEL: {{^}}s_fabs_f16:
 ; CI: flat_load_ushort [[VAL:v[0-9]+]],
-; CI: v_and_b32_e32 [[CVT0:v[0-9]+]], 0x7fff, [[VAL]]
+; CI: v_and_b32_e32 [[RESULT:v[0-9]+]], 0x7fff, [[VAL]]
 ; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @s_fabs_f16(half addrspace(1)* %out, half %in) {
   %fabs = call half @llvm.fabs.f16(half %in)
@@ -67,7 +67,7 @@ define amdgpu_kernel void @s_fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI:     v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
-; GCN: flat_store_dwordx2
+; GCN: {{flat|global}}_store_dwordx2
 define amdgpu_kernel void @s_fabs_v4f16(<4 x half> addrspace(1)* %out, <4 x half> %in) {
   %fabs = call <4 x half> @llvm.fabs.v4f16(<4 x half> %in)
   store <4 x half> %fabs, <4 x half> addrspace(1)* %out
@@ -75,8 +75,8 @@ define amdgpu_kernel void @s_fabs_v4f16(<4 x half> addrspace(1)* %out, <4 x half
 }
 
 ; GCN-LABEL: {{^}}fabs_fold_f16:
-; GCN: flat_load_ushort [[IN0:v[0-9]+]]
-; GCN: flat_load_ushort [[IN1:v[0-9]+]]
+; GCN: {{flat|global}}_load_ushort [[IN0:v[0-9]+]]
+; GCN: {{flat|global}}_load_ushort [[IN1:v[0-9]+]]
 
 ; CI-DAG: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[IN0]]
 ; CI-DAG: v_cvt_f32_f16_e64 [[ABS_CVT1:v[0-9]+]], |[[IN1]]|
@@ -95,7 +95,7 @@ define amdgpu_kernel void @fabs_fold_f16(half addrspace(1)* %out, half %in0, hal
 }
 
 ; GCN-LABEL: {{^}}v_fabs_v2f16:
-; GCN: flat_load_dword [[VAL:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 ; GCN: v_and_b32_e32 v{{[0-9]+}}, 0x7fff7fff, [[VAL]]
 define amdgpu_kernel void @v_fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -118,7 +118,7 @@ define amdgpu_kernel void @fabs_free_v2f16(<2 x half> addrspace(1)* %out, i32 %i
 }
 
 ; GCN-LABEL: {{^}}v_fabs_fold_v2f16:
-; GCN: flat_load_dword [[VAL:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 
 ; CI: v_cvt_f32_f16_e32
 ; CI: v_cvt_f32_f16_e32
@@ -127,8 +127,7 @@ define amdgpu_kernel void @fabs_free_v2f16(<2 x half> addrspace(1)* %out, i32 %i
 ; CI: v_mul_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}|, v{{[0-9]+}}
 ; CI: v_cvt_f16_f32
 
-; VI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_mul_f16_sdwa v{{[0-9]+}}, |v{{[0-9]+}}|, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI: v_mul_f16_sdwa v{{[0-9]+}}, |v{{[0-9]+}}|, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI: v_mul_f16_e64 v{{[0-9]+}}, |v{{[0-9]+}}|, v{{[0-9]+}}
 
 ; GFX9: v_and_b32_e32 [[FABS:v[0-9]+]], 0x7fff7fff, [[VAL]]
diff --git a/test/CodeGen/AMDGPU/fabs.ll b/test/CodeGen/AMDGPU/fabs.ll
index 600c6cd8230eb..550ad7956c929 100644
--- a/test/CodeGen/AMDGPU/fabs.ll
+++ b/test/CodeGen/AMDGPU/fabs.ll
@@ -83,7 +83,7 @@ define amdgpu_kernel void @fabs_fn_fold(float addrspace(1)* %out, float %in0, fl
   ret void
 }
 
-; GCN-LABEL: {{^}}fabs_fold:
+; FUNC-LABEL: {{^}}fabs_fold:
 ; SI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
 ; VI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c
 ; GCN-NOT: and
@@ -95,6 +95,18 @@ define amdgpu_kernel void @fabs_fold(float addrspace(1)* %out, float %in0, float
   ret void
 }
 
+; Make sure we turn some integer operations back into fabs
+; FUNC-LABEL: {{^}}bitpreserve_fabs_f32:
+; GCN: v_add_f32_e64 v{{[0-9]+}}, |s{{[0-9]+}}|, 1.0
+define amdgpu_kernel void @bitpreserve_fabs_f32(float addrspace(1)* %out, float %in) {
+  %in.bc = bitcast float %in to i32
+  %int.abs = and i32 %in.bc, 2147483647
+  %bc = bitcast i32 %int.abs to float
+  %fadd = fadd float %bc, 1.0
+  store float %fadd, float addrspace(1)* %out
+  ret void
+}
+
 declare float @fabs(float) readnone
 declare float @llvm.fabs.f32(float) readnone
 declare <2 x float> @llvm.fabs.v2f32(<2 x float>) readnone
diff --git a/test/CodeGen/AMDGPU/fadd.f16.ll b/test/CodeGen/AMDGPU/fadd.f16.ll
index 88b3be0e0d31c..447c253afa944 100644
--- a/test/CodeGen/AMDGPU/fadd.f16.ll
+++ b/test/CodeGen/AMDGPU/fadd.f16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}fadd_f16
@@ -60,8 +60,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}fadd_v2f16:
-; GCN: {{buffer|flat}}_load_dword v[[A_V2_F16:[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword v[[B_V2_F16:[0-9]+]]
+; SI: buffer_load_dword v[[A_V2_F16:[0-9]+]]
+; SI: buffer_load_dword v[[B_V2_F16:[0-9]+]]
+; VI: flat_load_dword v[[B_V2_F16:[0-9]+]]
+; VI: flat_load_dword v[[A_V2_F16:[0-9]+]]
 
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
 ; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
diff --git a/test/CodeGen/AMDGPU/fadd.ll b/test/CodeGen/AMDGPU/fadd.ll
index 621a0de281db5..a2f1f7195f275 100644
--- a/test/CodeGen/AMDGPU/fadd.ll
+++ b/test/CodeGen/AMDGPU/fadd.ll
@@ -72,4 +72,4 @@ define amdgpu_kernel void @fadd_0_nsz_attr_f32(float addrspace(1)* %out, float %
 }
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind "no-signed-zeros-fp-math"="true" }
-\ No newline at end of file
+attributes #1 = { nounwind "no-signed-zeros-fp-math"="true" }
diff --git a/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index 5ffa45595e701..48e4828557e5b 100644
--- a/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -1,10 +1,11 @@
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=GCN-FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=-fp32-denormals,+fp-exceptions < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-EXCEPT -check-prefix=VI -check-prefix=GCN-FLUSH %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs -mattr=+fp32-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX9-DENORM %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=-fp32-denormals,+fp-exceptions < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-EXCEPT,VI,GCN-FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-DENORM,GCN-DENORM %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=-fp32-denormals < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX9-FLUSH,GCN-FLUSH %s
 
 ; GCN-LABEL: {{^}}test_no_fold_canonicalize_loaded_value_f32:
-; GCN: v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
+; GCN-FLUSH:   v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
+; GFX9-DENORM: v_max_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @test_no_fold_canonicalize_loaded_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
@@ -16,7 +17,7 @@ define amdgpu_kernel void @test_no_fold_canonicalize_loaded_value_f32(float addr
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_fmul_value_f32:
 ; GCN: v_mul_f32_e32 [[V:v[0-9]+]], 0x41700000, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fmul_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -30,7 +31,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fmul_value_f32(float addrspace
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_sub_value_f32:
 ; GCN: v_sub_f32_e32 [[V:v[0-9]+]], 0x41700000, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_sub_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -44,7 +45,7 @@ define amdgpu_kernel void @test_fold_canonicalize_sub_value_f32(float addrspace(
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_add_value_f32:
 ; GCN: v_add_f32_e32 [[V:v[0-9]+]], 0x41700000, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_add_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -58,7 +59,7 @@ define amdgpu_kernel void @test_fold_canonicalize_add_value_f32(float addrspace(
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_sqrt_value_f32:
 ; GCN: v_sqrt_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_sqrt_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -72,7 +73,7 @@ define amdgpu_kernel void @test_fold_canonicalize_sqrt_value_f32(float addrspace
 
 ; GCN-LABEL: test_fold_canonicalize_fceil_value_f32:
 ; GCN: v_ceil_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fceil_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -86,7 +87,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fceil_value_f32(float addrspac
 
 ; GCN-LABEL: test_fold_canonicalize_floor_value_f32:
 ; GCN: v_floor_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_floor_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -100,7 +101,7 @@ define amdgpu_kernel void @test_fold_canonicalize_floor_value_f32(float addrspac
 
 ; GCN-LABEL: test_fold_canonicalize_fma_value_f32:
 ; GCN: v_fma_f32 [[V:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fma_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -115,7 +116,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fma_value_f32(float addrspace(
 ; GCN-LABEL: test_fold_canonicalize_fmuladd_value_f32:
 ; GCN-FLUSH: v_mac_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9-DENORM: v_fma_f32 [[V:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fmuladd_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -128,9 +129,10 @@ define amdgpu_kernel void @test_fold_canonicalize_fmuladd_value_f32(float addrsp
 }
 
 ; GCN-LABEL: test_fold_canonicalize_canonicalize_value_f32:
-; GCN: flat_load_dword [[LOAD:v[0-9]+]],
-; GCN: v_mul_f32_e32 [[V:v[0-9]+]], 1.0, [[LOAD]]
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_load_dword [[LOAD:v[0-9]+]],
+; GCN-FLUSH:  v_mul_f32_e32 [[V:v[0-9]+]], 1.0, [[LOAD]]
+; GCN-DENORM: v_max_f32_e32 [[V:v[0-9]+]], [[LOAD]], [[LOAD]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_canonicalize_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -144,7 +146,7 @@ define amdgpu_kernel void @test_fold_canonicalize_canonicalize_value_f32(float a
 
 ; GCN-LABEL: test_fold_canonicalize_fpextend_value_f64_f32:
 ; GCN: v_cvt_f64_f32_e32 [[V:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
-; GCN: flat_store_dwordx2 v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dwordx2 v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fpextend_value_f64_f32(float addrspace(1)* %arg, double addrspace(1)* %out) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -159,7 +161,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fpextend_value_f64_f32(float a
 
 ; GCN-LABEL: test_fold_canonicalize_fpextend_value_f32_f16:
 ; GCN: v_cvt_f32_f16_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fpextend_value_f32_f16(half addrspace(1)* %arg, float addrspace(1)* %out) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -174,7 +176,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fpextend_value_f32_f16(half ad
 
 ; GCN-LABEL: test_fold_canonicalize_fpround_value_f32_f64:
 ; GCN: v_cvt_f32_f64_e32 [[V:v[0-9]+]], v[{{[0-9:]+}}]
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fpround_value_f32_f64(double addrspace(1)* %arg, float addrspace(1)* %out) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -189,7 +191,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fpround_value_f32_f64(double a
 
 ; GCN-LABEL: test_fold_canonicalize_fpround_value_f16_f32:
 ; GCN: v_cvt_f16_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_short v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_short v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fpround_value_f16_f32(float addrspace(1)* %arg, half addrspace(1)* %out) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -209,7 +211,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fpround_value_f16_f32(float ad
 ; GFX9: v_cvt_f16_f32_e32 [[V1:v[0-9]+]], v{{[0-9]+}}
 ; GFX9: v_and_b32_e32 [[V0_16:v[0-9]+]], 0xffff, [[V0]]
 ; GFX9: v_lshl_or_b32 [[V:v[0-9]+]], [[V1]], 16, [[V0_16]]
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fpround_value_v2f16_v2f32(<2 x float> addrspace(1)* %arg, <2 x half> addrspace(1)* %out) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -223,7 +225,8 @@ define amdgpu_kernel void @test_fold_canonicalize_fpround_value_v2f16_v2f32(<2 x
 }
 
 ; GCN-LABEL: test_no_fold_canonicalize_fneg_value_f32:
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, 1.0, -v{{[0-9]+}}
+; GCN-FLUSH:  v_mul_f32_e64 v{{[0-9]+}}, 1.0, -v{{[0-9]+}}
+; GCN-DENORM: v_max_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}
 define amdgpu_kernel void @test_no_fold_canonicalize_fneg_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
@@ -236,7 +239,7 @@ define amdgpu_kernel void @test_no_fold_canonicalize_fneg_value_f32(float addrsp
 
 ; GCN-LABEL: test_fold_canonicalize_fneg_value_f32:
 ; GCN: v_xor_b32_e32 [[V:v[0-9]+]], 0x80000000, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fneg_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -250,7 +253,8 @@ define amdgpu_kernel void @test_fold_canonicalize_fneg_value_f32(float addrspace
 }
 
 ; GCN-LABEL: test_no_fold_canonicalize_fabs_value_f32:
-; GCN: v_mul_f32_e64 v{{[0-9]+}}, 1.0, |v{{[0-9]+}}|
+; GCN-FLUSH:  v_mul_f32_e64 v{{[0-9]+}}, 1.0, |v{{[0-9]+}}|
+; GCN-DENORM: v_max_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}|, |v{{[0-9]+}}|
 define amdgpu_kernel void @test_no_fold_canonicalize_fabs_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
@@ -263,7 +267,7 @@ define amdgpu_kernel void @test_no_fold_canonicalize_fabs_value_f32(float addrsp
 
 ; GCN-LABEL: test_fold_canonicalize_fabs_value_f32:
 ; GCN: v_and_b32_e32 [[V:v[0-9]+]], 0x7fffffff, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_fabs_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -278,7 +282,7 @@ define amdgpu_kernel void @test_fold_canonicalize_fabs_value_f32(float addrspace
 
 ; GCN-LABEL: test_fold_canonicalize_sin_value_f32:
 ; GCN: v_sin_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_sin_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -292,7 +296,7 @@ define amdgpu_kernel void @test_fold_canonicalize_sin_value_f32(float addrspace(
 
 ; GCN-LABEL: test_fold_canonicalize_cos_value_f32:
 ; GCN: v_cos_f32_e32 [[V:v[0-9]+]], v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_cos_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -307,7 +311,7 @@ define amdgpu_kernel void @test_fold_canonicalize_cos_value_f32(float addrspace(
 ; GCN-LABEL: test_fold_canonicalize_sin_value_f16:
 ; GCN: v_sin_f32_e32 [[V0:v[0-9]+]], v{{[0-9]+}}
 ; GCN: v_cvt_f16_f32_e32 [[V:v[0-9]+]], [[V0]]
-; GCN: flat_store_short v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_short v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_sin_value_f16(half addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -322,7 +326,7 @@ define amdgpu_kernel void @test_fold_canonicalize_sin_value_f16(half addrspace(1
 ; GCN-LABEL: test_fold_canonicalize_cos_value_f16:
 ; GCN: v_cos_f32_e32 [[V0:v[0-9]+]], v{{[0-9]+}}
 ; GCN: v_cvt_f16_f32_e32 [[V:v[0-9]+]], [[V0]]
-; GCN: flat_store_short v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_short v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_cos_value_f16(half addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -336,7 +340,7 @@ define amdgpu_kernel void @test_fold_canonicalize_cos_value_f16(half addrspace(1
 
 ; GCN-LABEL: test_fold_canonicalize_qNaN_value_f32:
 ; GCN: v_mov_b32_e32 [[V:v[0-9]+]], 0x7fc00000
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_qNaN_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -349,7 +353,7 @@ define amdgpu_kernel void @test_fold_canonicalize_qNaN_value_f32(float addrspace
 ; GCN-LABEL: test_fold_canonicalize_minnum_value_from_load_f32:
 ; VI: v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
 ; GFX9: v_min_f32_e32 [[V:v[0-9]+]], 0, v{{[0-9]+}}
-; GFX9: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GFX9: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 define amdgpu_kernel void @test_fold_canonicalize_minnum_value_from_load_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
@@ -362,7 +366,7 @@ define amdgpu_kernel void @test_fold_canonicalize_minnum_value_from_load_f32(flo
 
 ; GCN-LABEL: test_fold_canonicalize_minnum_value_f32:
 ; GCN: v_min_f32_e32 [[V:v[0-9]+]], 0, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_minnum_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -377,8 +381,9 @@ define amdgpu_kernel void @test_fold_canonicalize_minnum_value_f32(float addrspa
 
 ; GCN-LABEL: test_fold_canonicalize_sNaN_value_f32:
 ; GCN:  v_min_f32_e32 [[V0:v[0-9]+]], 0x7f800001, v{{[0-9]+}}
-; GCN:  v_mul_f32_e32 v{{[0-9]+}}, 1.0, [[V0]]
-; GCN:  flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN-FLUSH:  v_mul_f32_e32 [[RESULT:v[0-9]+]], 1.0, [[V0]]
+; GCN-DENORM: v_max_f32_e32 [[RESULT:v[0-9]+]], [[V0]], [[V0]]
+; GCN:  {{flat|global}}_store_dword v[{{[0-9:]+}}], [[RESULT]]
 define amdgpu_kernel void @test_fold_canonicalize_sNaN_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
@@ -390,10 +395,10 @@ define amdgpu_kernel void @test_fold_canonicalize_sNaN_value_f32(float addrspace
 }
 
 ; GCN-LABEL: test_fold_canonicalize_denorm_value_f32:
-; GFX9:  v_min_f32_e32 [[V:v[0-9]+]], 0x7fffff, v{{[0-9]+}}
+; GFX9:  v_min_f32_e32 [[RESULT:v[0-9]+]], 0x7fffff, v{{[0-9]+}}
 ; VI:    v_min_f32_e32 [[V0:v[0-9]+]], 0x7fffff, v{{[0-9]+}}
-; VI:    v_mul_f32_e32 v{{[0-9]+}}, 1.0, [[V0]]
-; GCN:   flat_store_dword v[{{[0-9:]+}}], [[V]]
+; VI:    v_mul_f32_e32 [[RESULT:v[0-9]+]], 1.0, [[V0]]
+; GCN:   {{flat|global}}_store_dword v[{{[0-9:]+}}], [[RESULT]]
 ; GFX9-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_denorm_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -406,10 +411,10 @@ define amdgpu_kernel void @test_fold_canonicalize_denorm_value_f32(float addrspa
 }
 
 ; GCN-LABEL: test_fold_canonicalize_maxnum_value_from_load_f32:
-; GFX9:  v_max_f32_e32 [[V:v[0-9]+]], 0, v{{[0-9]+}}
+; GFX9:  v_max_f32_e32 [[RESULT:v[0-9]+]], 0, v{{[0-9]+}}
 ; VI:    v_max_f32_e32 [[V0:v[0-9]+]], 0, v{{[0-9]+}}
-; VI:    v_mul_f32_e32 v{{[0-9]+}}, 1.0, [[V0]]
-; GCN:  flat_store_dword v[{{[0-9:]+}}], [[V]]
+; VI:    v_mul_f32_e32 [[RESULT:v[0-9]+]], 1.0, [[V0]]
+; GCN:  {{flat|global}}_store_dword v[{{[0-9:]+}}], [[RESULT]]
 ; GFX9-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_from_load_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -423,7 +428,7 @@ define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_from_load_f32(flo
 
 ; GCN-LABEL: test_fold_canonicalize_maxnum_value_f32:
 ; GCN: v_max_f32_e32 [[V:v[0-9]+]], 0, v{{[0-9]+}}
-; GCN: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dword v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_f32(float addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -438,7 +443,7 @@ define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_f32(float addrspa
 
 ; GCN-LABEL: test_fold_canonicalize_maxnum_value_f64:
 ; GCN: v_max_f64 [[V:v\[[0-9]+:[0-9]+\]]], v[{{[0-9:]+}}], 0
-; GCN: flat_store_dwordx2 v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_store_dwordx2 v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_maxnum_value_f64(double addrspace(1)* %arg) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -472,8 +477,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_load_nnan_value_f32
-; GFX9-DENORM: flat_load_dword [[V:v[0-9]+]],
-; GFX9-DENORM: flat_store_dword v[{{[0-9:]+}}], [[V]]
+; GFX9-DENORM: global_load_dword [[V:v[0-9]+]],
+; GFX9-DENORM: global_store_dword v[{{[0-9:]+}}], [[V]]
 ; GFX9-DENORM-NOT: 1.0
 ; GCN-FLUSH: v_mul_f32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
 define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f32(float addrspace(1)* %arg, float addrspace(1)* %out) #1 {
@@ -487,8 +492,8 @@ define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f32(float addr
 }
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_load_nnan_value_f64
-; GCN: flat_load_dwordx2 [[V:v\[[0-9:]+\]]],
-; GCN: flat_store_dwordx2 v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_load_dwordx2 [[V:v\[[0-9:]+\]]],
+; GCN: {{flat|global}}_store_dwordx2 v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f64(double addrspace(1)* %arg, double addrspace(1)* %out) #1 {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -501,8 +506,8 @@ define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f64(double add
 }
 
 ; GCN-LABEL: {{^}}test_fold_canonicalize_load_nnan_value_f16
-; GCN: flat_load_ushort [[V:v[0-9]+]],
-; GCN: flat_store_short v[{{[0-9:]+}}], [[V]]
+; GCN: {{flat|global}}_load_ushort [[V:v[0-9]+]],
+; GCN: {{flat|global}}_store_short v[{{[0-9:]+}}], [[V]]
 ; GCN-NOT: 1.0
 define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f16(half addrspace(1)* %arg, half addrspace(1)* %out) #1 {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -514,6 +519,10 @@ define amdgpu_kernel void @test_fold_canonicalize_load_nnan_value_f16(half addrs
   ret void
 }
 
+; Avoid failing the test on FreeBSD11.0 which will match the GCN-NOT: 1.0
+; in the .amd_amdgpu_isa "amdgcn-unknown-freebsd11.0--gfx802" directive
+; CHECK: .amd_amdgpu_isa
+
 declare float @llvm.canonicalize.f32(float) #0
 declare double @llvm.canonicalize.f64(double) #0
 declare half @llvm.canonicalize.f16(half) #0
diff --git a/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index dd8e277c1c75f..24195660e576a 100644
--- a/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 declare half @llvm.fabs.f16(half) #0
 declare half @llvm.canonicalize.f16(half) #0
@@ -9,7 +9,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_f16:
-; GCN: v_mul_f16_e32 [[REG:v[0-9]+]], 1.0, {{v[0-9]+}}
+; GCN: v_max_f16_e32 [[REG:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
 ; GCN: buffer_store_short [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
@@ -19,7 +19,7 @@ define amdgpu_kernel void @v_test_canonicalize_var_f16(half addrspace(1)* %out)
 }
 
 ; GCN-LABEL: {{^}}s_test_canonicalize_var_f16:
-; GCN: v_mul_f16_e64 [[REG:v[0-9]+]], 1.0, {{s[0-9]+}}
+; GCN: v_max_f16_e64 [[REG:v[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
 ; GCN: buffer_store_short [[REG]]
 define amdgpu_kernel void @s_test_canonicalize_var_f16(half addrspace(1)* %out, i16 zeroext %val.arg) #1 {
   %val = bitcast i16 %val.arg to half
@@ -29,7 +29,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f16(half addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fabs_var_f16:
-; GCN: v_mul_f16_e64 [[REG:v[0-9]+]], 1.0, |{{v[0-9]+}}|
+; GCN: v_max_f16_e64 [[REG:v[0-9]+]], |{{v[0-9]+}}|, |{{v[0-9]+}}|
 ; GCN: buffer_store_short [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fabs_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
@@ -40,7 +40,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f16(half addrspace(1)* %
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_fabs_var_f16:
-; GCN: v_mul_f16_e64 [[REG:v[0-9]+]], 1.0, -|{{v[0-9]+}}|
+; GCN: v_max_f16_e64 [[REG:v[0-9]+]], -|{{v[0-9]+}}|, -|{{v[0-9]+}}|
 ; GCN: buffer_store_short [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
@@ -52,7 +52,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f16(half addrspace(
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_var_f16:
-; GCN: v_mul_f16_e64 [[REG:v[0-9]+]], 1.0, -{{v[0-9]+}}
+; GCN: v_max_f16_e64 [[REG:v[0-9]+]], -{{v[0-9]+}}, -{{v[0-9]+}}
 ; GCN: buffer_store_short [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_f16(half addrspace(1)* %out) #1 {
   %val = load half, half addrspace(1)* %out
@@ -207,12 +207,11 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f16(half addrspace
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_v2f16:
-; VI: v_mov_b32_e32 v[[CONST1:[0-9]+]], 0x3c00
-; VI-DAG: v_mul_f16_sdwa [[REG0:v[0-9]+]], v[[CONST1]], {{v[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mul_f16_e32 [[REG1:v[0-9]+]], 1.0, {{v[0-9]+}}
+; VI-DAG: v_max_f16_sdwa [[REG0:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_max_f16_e32 [[REG1:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}}
 ; VI-NOT: v_and_b32
 
-; GFX9: v_pk_mul_f16 [[REG:v[0-9]+]], 1.0, {{v[0-9]+$}}
+; GFX9: v_pk_max_f16 [[REG:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+$}}
 ; GFX9: buffer_store_dword [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_var_v2f16(<2 x half> addrspace(1)* %out) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -227,14 +226,13 @@ define amdgpu_kernel void @v_test_canonicalize_var_v2f16(<2 x half> addrspace(1)
 ; GCN-LABEL: {{^}}v_test_canonicalize_fabs_var_v2f16:
 ; VI-DAG: v_bfe_u32
 ; VI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0x7fff7fff, v{{[0-9]+}}
-; VI-DAG: v_mov_b32_e32 v[[CONST1:[0-9]+]], 0x3c00
-; VI: v_mul_f16_sdwa [[REG0:v[0-9]+]], v[[CONST1]], v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI: v_mul_f16_e32 [[REG1:v[0-9]+]], 1.0, v{{[0-9]+}}
+; VI: v_max_f16_sdwa [[REG0:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI: v_max_f16_e32 [[REG1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-NOT: 0xffff
 ; VI: v_or_b32
 
 ; GFX9: v_and_b32_e32 [[ABS:v[0-9]+]], 0x7fff7fff, v{{[0-9]+}}
-; GFX9: v_pk_mul_f16 [[REG:v[0-9]+]], 1.0, [[ABS]]{{$}}
+; GFX9: v_pk_max_f16 [[REG:v[0-9]+]], [[ABS]], [[ABS]]{{$}}
 ; GCN: buffer_store_dword
 define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2f16(<2 x half> addrspace(1)* %out) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -247,14 +245,13 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2f16(<2 x half> addrspa
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_fabs_var_v2f16:
-; VI-DAG: v_mov_b32_e32 v[[CONST1:[0-9]+]], 0x3c00
 ; VI-DAG: v_or_b32_e32 v{{[0-9]+}}, 0x80008000, v{{[0-9]+}}
-; VI-DAG: v_mul_f16_sdwa [[REG0:v[0-9]+]], v[[CONST1]], v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mul_f16_e32 [[REG1:v[0-9]+]], 1.0, v{{[0-9]+}}
+; VI-DAG: v_max_f16_sdwa [[REG0:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_max_f16_e32 [[REG1:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}}
 ; VI: v_or_b32
 
 ; GFX9: v_and_b32_e32 [[ABS:v[0-9]+]], 0x7fff7fff, v{{[0-9]+}}
-; GFX9: v_pk_mul_f16 [[REG:v[0-9]+]], 1.0, [[ABS]] neg_lo:[0,1] neg_hi:[0,1]{{$}}
+; GFX9: v_pk_max_f16 [[REG:v[0-9]+]], [[ABS]], [[ABS]] neg_lo:[1,1] neg_hi:[1,1]{{$}}
 ; GCN: buffer_store_dword
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2f16(<2 x half> addrspace(1)* %out) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -267,15 +264,13 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2f16(<2 x half> ad
   ret void
 }
 
-; FIXME: Fold modifier
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_var_v2f16:
-; VI-DAG: v_mov_b32_e32 v[[CONST1:[0-9]+]], 0x3c00
-; VI-DAG: v_xor_b32_e32 [[FNEG:v[0-9]+]], 0x80008000, v{{[0-9]+}}
-; VI-DAG: v_mul_f16_sdwa [[REG1:v[0-9]+]], v[[CONST1]], [[FNEG]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-DAG: v_mul_f16_e32 [[REG0:v[0-9]+]], 1.0, [[FNEG]]
+; VI:     v_xor_b32_e32 [[FNEG:v[0-9]+]], 0x80008000, v{{[0-9]+}}
+; VI-DAG: v_max_f16_sdwa [[REG1:v[0-9]+]], [[FNEG]], [[FNEG]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI-DAG: v_max_f16_e32 [[REG0:v[0-9]+]], [[FNEG]], [[FNEG]]
 ; VI-NOT: 0xffff
 
-; GFX9: v_pk_mul_f16 [[REG:v[0-9]+]], 1.0, {{v[0-9]+}} neg_lo:[0,1] neg_hi:[0,1]{{$}}
+; GFX9: v_pk_max_f16 [[REG:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}} neg_lo:[1,1] neg_hi:[1,1]{{$}}
 ; GFX9: buffer_store_dword [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2f16(<2 x half> addrspace(1)* %out) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -288,12 +283,11 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2f16(<2 x half> addrspa
 }
 
 ; GCN-LABEL: {{^}}s_test_canonicalize_var_v2f16:
-; VI: v_mov_b32_e32 [[ONE:v[0-9]+]], 0x3c00
-; VI: v_mul_f16_sdwa [[REG0:v[0-9]+]], [[ONE]], {{v[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI: v_mul_f16_e64 [[REG1:v[0-9]+]], 1.0, {{s[0-9]+}}
+; VI: v_max_f16_sdwa [[REG0:v[0-9]+]], {{v[0-9]+}}, {{v[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI: v_max_f16_e64 [[REG1:v[0-9]+]], {{s[0-9]+}}, {{s[0-9]+}}
 ; VI-NOT: v_and_b32
 
-; GFX9: v_pk_mul_f16 [[REG:v[0-9]+]], 1.0, {{s[0-9]+$}}
+; GFX9: v_pk_max_f16 [[REG:v[0-9]+]], {{s[0-9]+}}, {{s[0-9]+$}}
 ; GFX9: buffer_store_dword [[REG]]
 define amdgpu_kernel void @s_test_canonicalize_var_v2f16(<2 x half> addrspace(1)* %out, i32 zeroext %val.arg) #1 {
   %val = bitcast i32 %val.arg to <2 x half>
diff --git a/test/CodeGen/AMDGPU/fcanonicalize.ll b/test/CodeGen/AMDGPU/fcanonicalize.ll
index feb4c7bd4a183..c0ac6599e4a0e 100644
--- a/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -1,9 +1,12 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 declare float @llvm.fabs.f32(float) #0
 declare float @llvm.canonicalize.f32(float) #0
 declare double @llvm.fabs.f64(double) #0
 declare double @llvm.canonicalize.f64(double) #0
+declare half @llvm.canonicalize.f16(half) #0
+declare <2 x half> @llvm.canonicalize.v2f16(<2 x half>) #0
+declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_f32:
 ; GCN: v_mul_f32_e32 [[REG:v[0-9]+]], 1.0, {{v[0-9]+}}
@@ -203,7 +206,7 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f32(float addrspac
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_var_f64:
-; GCN: v_mul_f64 [[REG:v\[[0-9]+:[0-9]+\]]], 1.0, {{v\[[0-9]+:[0-9]+\]}}
+; GCN: v_max_f64 [[REG:v\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}
 ; GCN: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
@@ -213,7 +216,7 @@ define amdgpu_kernel void @v_test_canonicalize_var_f64(double addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}s_test_canonicalize_var_f64:
-; GCN: v_mul_f64 [[REG:v\[[0-9]+:[0-9]+\]]], 1.0, {{s\[[0-9]+:[0-9]+\]}}
+; GCN: v_max_f64 [[REG:v\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}
 ; GCN: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @s_test_canonicalize_var_f64(double addrspace(1)* %out, double %val) #1 {
   %canonicalized = call double @llvm.canonicalize.f64(double %val)
@@ -222,7 +225,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f64(double addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fabs_var_f64:
-; GCN: v_mul_f64 [[REG:v\[[0-9]+:[0-9]+\]]], 1.0, |{{v\[[0-9]+:[0-9]+\]}}|
+; GCN: v_max_f64 [[REG:v\[[0-9]+:[0-9]+\]]], |{{v\[[0-9]+:[0-9]+\]}}|, |{{v\[[0-9]+:[0-9]+\]}}|
 ; GCN: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
@@ -233,7 +236,7 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_f64(double addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_fabs_var_f64:
-; GCN: v_mul_f64 [[REG:v\[[0-9]+:[0-9]\]]], 1.0, -|{{v\[[0-9]+:[0-9]+\]}}|
+; GCN: v_max_f64 [[REG:v\[[0-9]+:[0-9]\]]], -|{{v\[[0-9]+:[0-9]+\]}}|, -|{{v\[[0-9]+:[0-9]+\]}}|
 ; GCN: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
@@ -245,7 +248,7 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_f64(double addrspac
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_fneg_var_f64:
-; GCN: v_mul_f64 [[REG:v\[[0-9]+:[0-9]+\]]], 1.0, -{{v\[[0-9]+:[0-9]+\]}}
+; GCN: v_max_f64 [[REG:v\[[0-9]+:[0-9]+\]]], -{{v\[[0-9]+:[0-9]+\]}}, -{{v\[[0-9]+:[0-9]+\]}}
 ; GCN: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @v_test_canonicalize_fneg_var_f64(double addrspace(1)* %out) #1 {
   %val = load double, double addrspace(1)* %out
@@ -415,7 +418,120 @@ define amdgpu_kernel void @test_fold_canonicalize_snan3_value_f64(double addrspa
   ret void
 }
 
+; GCN-LABEL:  {{^}}test_canonicalize_value_f64_flush:
+; GCN: v_mul_f64 v[{{[0-9:]+}}], 1.0, v[{{[0-9:]+}}]
+define amdgpu_kernel void @test_canonicalize_value_f64_flush(double addrspace(1)* %arg, double addrspace(1)* %out) #4 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds double, double addrspace(1)* %arg, i32 %id
+  %v = load double, double addrspace(1)* %gep, align 8
+  %canonicalized = tail call double @llvm.canonicalize.f64(double %v)
+  %gep2 = getelementptr inbounds double, double addrspace(1)* %out, i32 %id
+  store double %canonicalized, double addrspace(1)* %gep2, align 8
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_f32_flush:
+; GCN: v_mul_f32_e32 {{v[0-9]+}}, 1.0, {{v[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_f32_flush(float addrspace(1)* %arg, float addrspace(1)* %out) #4 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
+  %v = load float, float addrspace(1)* %gep, align 4
+  %canonicalized = tail call float @llvm.canonicalize.f32(float %v)
+  %gep2 = getelementptr inbounds float, float addrspace(1)* %out, i32 %id
+  store float %canonicalized, float addrspace(1)* %gep2, align 4
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_f16_flush:
+; GCN: v_mul_f16_e32 {{v[0-9]+}}, 1.0, {{v[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_f16_flush(half addrspace(1)* %arg, half addrspace(1)* %out) #4 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds half, half addrspace(1)* %arg, i32 %id
+  %v = load half, half addrspace(1)* %gep, align 2
+  %canonicalized = tail call half @llvm.canonicalize.f16(half %v)
+  %gep2 = getelementptr inbounds half, half addrspace(1)* %out, i32 %id
+  store half %canonicalized, half addrspace(1)* %gep2, align 2
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_v2f16_flush_gfx8:
+; GCN:     v_mov_b32_e32 [[ONE:v[0-9]+]], 0x3c00
+; GCN-DAG: v_mul_f16_sdwa v{{[0-9]+}}, [[ONE]], v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GCN-DAG: v_mul_f16_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_v2f16_flush_gfx8(<2 x half> addrspace(1)* %arg, <2 x half> addrspace(1)* %out) #4 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %arg, i32 %id
+  %v = load <2 x half>, <2 x half> addrspace(1)* %gep, align 4
+  %canonicalized = tail call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %v)
+  %gep2 = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %out, i32 %id
+  store <2 x half> %canonicalized, <2 x half> addrspace(1)* %gep2, align 2
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_v2f16_flush_gfx9:
+; GCN-DAG: v_pk_mul_f16 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_v2f16_flush_gfx9(<2 x half> addrspace(1)* %arg, <2 x half> addrspace(1)* %out) #6 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %arg, i32 %id
+  %v = load <2 x half>, <2 x half> addrspace(1)* %gep, align 4
+  %canonicalized = tail call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %v)
+  %gep2 = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %out, i32 %id
+  store <2 x half> %canonicalized, <2 x half> addrspace(1)* %gep2, align 2
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_f64_denorm:
+; GCN: v_max_f64 v[{{[0-9:]+}}], v[{{[0-9:]+}}], v[{{[0-9:]+}}]
+define amdgpu_kernel void @test_canonicalize_value_f64_denorm(double addrspace(1)* %arg, double addrspace(1)* %out) #5 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds double, double addrspace(1)* %arg, i32 %id
+  %v = load double, double addrspace(1)* %gep, align 8
+  %canonicalized = tail call double @llvm.canonicalize.f64(double %v)
+  %gep2 = getelementptr inbounds double, double addrspace(1)* %out, i32 %id
+  store double %canonicalized, double addrspace(1)* %gep2, align 8
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_f32_denorm:
+; GCN: v_max_f32_e32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_f32_denorm(float addrspace(1)* %arg, float addrspace(1)* %out) #5 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds float, float addrspace(1)* %arg, i32 %id
+  %v = load float, float addrspace(1)* %gep, align 4
+  %canonicalized = tail call float @llvm.canonicalize.f32(float %v)
+  %gep2 = getelementptr inbounds float, float addrspace(1)* %out, i32 %id
+  store float %canonicalized, float addrspace(1)* %gep2, align 4
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_f16_denorm:
+; GCN: v_max_f16_e32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_f16_denorm(half addrspace(1)* %arg, half addrspace(1)* %out) #5 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds half, half addrspace(1)* %arg, i32 %id
+  %v = load half, half addrspace(1)* %gep, align 2
+  %canonicalized = tail call half @llvm.canonicalize.f16(half %v)
+  %gep2 = getelementptr inbounds half, half addrspace(1)* %out, i32 %id
+  store half %canonicalized, half addrspace(1)* %gep2, align 2
+  ret void
+}
+
+; GCN-LABEL:  {{^}}test_canonicalize_value_v2f16_denorm:
+; GCN: v_pk_max_f16 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(<2 x half> addrspace(1)* %arg, <2 x half> addrspace(1)* %out) #5 {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %arg, i32 %id
+  %v = load <2 x half>, <2 x half> addrspace(1)* %gep, align 4
+  %canonicalized = tail call <2 x half> @llvm.canonicalize.v2f16(<2 x half> %v)
+  %gep2 = getelementptr inbounds <2 x half>, <2 x half> addrspace(1)* %out, i32 %id
+  store <2 x half> %canonicalized, <2 x half> addrspace(1)* %gep2, align 2
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" }
 attributes #3 = { nounwind "target-features"="+fp32-denormals,+fp64-fp16-denormals" }
+attributes #4 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" "target-cpu"="tonga" }
+attributes #5 = { nounwind "target-features"="+fp32-denormals,+fp64-fp16-denormals" "target-cpu"="gfx900" }
+attributes #6 = { nounwind "target-features"="-fp32-denormals,-fp64-fp16-denormals" "target-cpu"="gfx900" }
diff --git a/test/CodeGen/AMDGPU/fcopysign.f16.ll b/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 8e984246cc94d..15d4d2a36676d 100644
--- a/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX8 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX8 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
 
 declare half @llvm.copysign.f16(half, half)
 declare float @llvm.copysign.f32(float, float)
@@ -12,15 +12,15 @@ declare <4 x half> @llvm.copysign.v4f16(<4 x half>, <4 x half>)
 declare i32 @llvm.amdgcn.workitem.id.x()
 
 ; GCN-LABEL: {{^}}test_copysign_f16:
-; SI: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
 ; SI: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
+; SI: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
 ; SI: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
 ; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
 ; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN_F32]]
 ; SI: v_cvt_f16_f32_e32 v[[OUT:[0-9]+]], v[[OUT_F32]]
-; GFX89: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
 ; GFX89: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
+; GFX89: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
 ; GFX89: s_movk_i32 s[[CONST:[0-9]+]], 0x7fff
 ; GFX89: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN]]
 ; GCN: buffer_store_short v[[OUT]]
@@ -38,8 +38,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f32_mag_f16_sign_f32:
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_dword v[[SIGN:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword v[[SIGN:[0-9]+]]
 ; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; GCN-DAG: v_cvt_f32_f16_e32 v[[MAG_EXT:[0-9]+]], v[[MAG]]
 ; GCN: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG_EXT]], v[[SIGN]]
@@ -62,8 +62,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f64_mag_f16_sign_f64:
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
 ; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; GCN-DAG: v_cvt_f32_f16_e32 v[[MAG_EXT:[0-9]+]], v[[MAG]]
 ; GCN-DAG: v_cvt_f64_f32_e32 v{{\[}}[[MAG_EXT_LO:[0-9]+]]:[[MAG_EXT_HI:[0-9]+]]{{\]}}, v[[MAG_EXT]]
@@ -87,8 +87,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f32_mag_f32_sign_f16:
-; GCN-DAG: {{buffer|flat}}_load_dword v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[SIGN:[0-9]+]]
 ; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
 ; SI: v_bfi_b32 v[[OUT:[0-9]+]], s[[CONST]], v[[MAG]], v[[SIGN_F32]]
@@ -113,8 +113,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f64_mag_f64_sign_f16:
-; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[MAG_LO:[0-9]+]]:[[MAG_HI:[0-9]+]]{{\]}}
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[MAG_LO:[0-9]+]]:[[MAG_HI:[0-9]+]]{{\]}}
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[SIGN:[0-9]+]]
 ; GCN-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
 ; SI: v_bfi_b32 v[[OUT_HI:[0-9]+]], s[[CONST]], v[[MAG_HI]], v[[SIGN_F32]]
@@ -139,8 +139,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f16_mag_f16_sign_f32:
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_dword v[[SIGN:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword v[[SIGN:[0-9]+]]
 ; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
 ; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN]]
@@ -167,8 +167,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f16_mag_f16_sign_f64:
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dwordx2 v{{\[}}[[SIGN_LO:[0-9]+]]:[[SIGN_HI:[0-9]+]]{{\]}}
 ; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f32_f16_e32 v[[MAG_F32:[0-9]+]], v[[MAG]]
 ; SI: v_bfi_b32 v[[OUT_F32:[0-9]+]], s[[CONST]], v[[MAG_F32]], v[[SIGN_HI]]
@@ -195,8 +195,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}test_copysign_out_f16_mag_f32_sign_f16:
-; GCN-DAG: {{buffer|flat}}_load_dword v[[MAG:[0-9]+]]
-; GCN-DAG: {{buffer|flat}}_load_ushort v[[SIGN:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword v[[MAG:[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_ushort v[[SIGN:[0-9]+]]
 ; SI-DAG: s_brev_b32 s[[CONST:[0-9]+]], -2
 ; SI-DAG: v_cvt_f16_f32_e32 v[[MAG_TRUNC:[0-9]+]], v[[MAG]]
 ; SI-DAG: v_cvt_f32_f16_e32 v[[SIGN_F32:[0-9]+]], v[[SIGN]]
diff --git a/test/CodeGen/AMDGPU/fcopysign.f64.ll b/test/CodeGen/AMDGPU/fcopysign.f64.ll
index 67779a8ff3b9b..161b7ad18e5db 100644
--- a/test/CodeGen/AMDGPU/fcopysign.f64.ll
+++ b/test/CodeGen/AMDGPU/fcopysign.f64.ll
@@ -24,7 +24,8 @@ define amdgpu_kernel void @test_copysign_f64(double addrspace(1)* %out, double %
 }
 
 ; FUNC-LABEL: {{^}}test_copysign_f64_f32:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[SMAG_LO:[0-9]+]]:[[SMAG_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}
+; SI-DAG: s_load_dwordx2 s{{\[}}[[SMAG_LO:[0-9]+]]:[[SMAG_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xb
+; VI-DAG: s_load_dwordx2 s{{\[}}[[SMAG_LO:[0-9]+]]:[[SMAG_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
 ; GCN-DAG: s_load_dword s[[SSIGN:[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}
 ; GCN-DAG: s_brev_b32 [[SCONST:s[0-9]+]], -2{{$}}
 ; GCN-DAG: v_mov_b32_e32 v[[VMAG_HI:[0-9]+]], s[[SMAG_HI]]
diff --git a/test/CodeGen/AMDGPU/fence-amdgiz.ll b/test/CodeGen/AMDGPU/fence-amdgiz.ll
index df675c9a8692e..0dd2a9241b230 100644
--- a/test/CodeGen/AMDGPU/fence-amdgiz.ll
+++ b/test/CodeGen/AMDGPU/fence-amdgiz.ll
@@ -1,13 +1,13 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgizcl -mcpu=kaveri < %s | FileCheck %s
 
 target datalayout = "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5"
-target triple = "amdgcn-amd-amdhsa-amdgizcl"
-
-; CHECK_LABEL: atomic_fence
-; CHECK: BB#0:
-; CHECK: ATOMIC_FENCE 4, 1
-; CHECK: s_endpgm
 
+; CHECK-LABEL: atomic_fence
+; CHECK:       %bb.0:
+; CHECK-NOT:   ATOMIC_FENCE
+; CHECK-NEXT:  s_waitcnt vmcnt(0)
+; CHECK-NEXT:  buffer_wbinvl1_vol
+; CHECK-NEXT:  s_endpgm
 define amdgpu_kernel void @atomic_fence() {
   fence acquire
   ret void
diff --git a/test/CodeGen/AMDGPU/fence-barrier.ll b/test/CodeGen/AMDGPU/fence-barrier.ll
new file mode 100644
index 0000000000000..f140a762942bd
--- /dev/null
+++ b/test/CodeGen/AMDGPU/fence-barrier.ll
@@ -0,0 +1,197 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -enable-si-insert-waitcnts=1 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN %s
+
+declare i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+declare i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.workgroup.id.x()
+declare void @llvm.amdgcn.s.barrier()
+
+@test_local.temp = internal addrspace(3) global [1 x i32] undef, align 4
+@test_global_local.temp = internal addrspace(3) global [1 x i32] undef, align 4
+
+; GCN-LABEL: {{^}}test_local
+; GCN: v_mov_b32_e32 v[[VAL:[0-9]+]], 0x777
+; GCN: ds_write_b32 v{{[0-9]+}}, v[[VAL]]
+; GCN: s_waitcnt lgkmcnt(0){{$}}
+; GCN-NEXT: s_barrier
+; GCN: flat_store_dword
+define amdgpu_kernel void @test_local(i32 addrspace(1)*) {
+  %2 = alloca i32 addrspace(1)*, align 4
+  store i32 addrspace(1)* %0, i32 addrspace(1)** %2, align 4
+  %3 = call i32 @llvm.amdgcn.workitem.id.x()
+  %4 = zext i32 %3 to i64
+  %5 = icmp eq i64 %4, 0
+  br i1 %5, label %6, label %7
+
+; <label>:6:                                      ; preds = %1
+  store i32 1911, i32 addrspace(3)* getelementptr inbounds ([1 x i32], [1 x i32] addrspace(3)* @test_local.temp, i64 0, i64 0), align 4
+  br label %7
+
+; <label>:7:                                      ; preds = %6, %1
+  fence syncscope("workgroup") release
+  call void @llvm.amdgcn.s.barrier()
+  fence syncscope("workgroup") acquire
+  %8 = load i32, i32 addrspace(3)* getelementptr inbounds ([1 x i32], [1 x i32] addrspace(3)* @test_local.temp, i64 0, i64 0), align 4
+  %9 = load i32 addrspace(1)*, i32 addrspace(1)** %2, align 4
+  %10 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %11 = call i32 @llvm.amdgcn.workitem.id.x()
+  %12 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %13 = getelementptr inbounds i8, i8 addrspace(2)* %10, i64 4
+  %14 = bitcast i8 addrspace(2)* %13 to i16 addrspace(2)*
+  %15 = load i16, i16 addrspace(2)* %14, align 4
+  %16 = zext i16 %15 to i32
+  %17 = mul i32 %12, %16
+  %18 = add i32 %17, %11
+  %19 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %20 = zext i32 %18 to i64
+  %21 = bitcast i8 addrspace(2)* %19 to i64 addrspace(2)*
+  %22 = load i64, i64 addrspace(2)* %21, align 8
+  %23 = add i64 %22, %20
+  %24 = getelementptr inbounds i32, i32 addrspace(1)* %9, i64 %23
+  store i32 %8, i32 addrspace(1)* %24, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_global
+; GCN: v_add_u32_e32 v{{[0-9]+}}, vcc, 0x888, v{{[0-9]+}}
+; GCN: flat_store_dword
+; GCN: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NEXT: s_barrier
+define amdgpu_kernel void @test_global(i32 addrspace(1)*) {
+  %2 = alloca i32 addrspace(1)*, align 4
+  %3 = alloca i32, align 4
+  store i32 addrspace(1)* %0, i32 addrspace(1)** %2, align 4
+  store i32 0, i32* %3, align 4
+  br label %4
+
+; <label>:4:                                      ; preds = %58, %1
+  %5 = load i32, i32* %3, align 4
+  %6 = sext i32 %5 to i64
+  %7 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %8 = call i32 @llvm.amdgcn.workitem.id.x()
+  %9 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %10 = getelementptr inbounds i8, i8 addrspace(2)* %7, i64 4
+  %11 = bitcast i8 addrspace(2)* %10 to i16 addrspace(2)*
+  %12 = load i16, i16 addrspace(2)* %11, align 4
+  %13 = zext i16 %12 to i32
+  %14 = mul i32 %9, %13
+  %15 = add i32 %14, %8
+  %16 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %17 = zext i32 %15 to i64
+  %18 = bitcast i8 addrspace(2)* %16 to i64 addrspace(2)*
+  %19 = load i64, i64 addrspace(2)* %18, align 8
+  %20 = add i64 %19, %17
+  %21 = icmp ult i64 %6, %20
+  br i1 %21, label %22, label %61
+
+; <label>:22:                                     ; preds = %4
+  %23 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %24 = call i32 @llvm.amdgcn.workitem.id.x()
+  %25 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %26 = getelementptr inbounds i8, i8 addrspace(2)* %23, i64 4
+  %27 = bitcast i8 addrspace(2)* %26 to i16 addrspace(2)*
+  %28 = load i16, i16 addrspace(2)* %27, align 4
+  %29 = zext i16 %28 to i32
+  %30 = mul i32 %25, %29
+  %31 = add i32 %30, %24
+  %32 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %33 = zext i32 %31 to i64
+  %34 = bitcast i8 addrspace(2)* %32 to i64 addrspace(2)*
+  %35 = load i64, i64 addrspace(2)* %34, align 8
+  %36 = add i64 %35, %33
+  %37 = add i64 %36, 2184
+  %38 = trunc i64 %37 to i32
+  %39 = load i32 addrspace(1)*, i32 addrspace(1)** %2, align 4
+  %40 = load i32, i32* %3, align 4
+  %41 = sext i32 %40 to i64
+  %42 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %43 = call i32 @llvm.amdgcn.workitem.id.x()
+  %44 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %45 = getelementptr inbounds i8, i8 addrspace(2)* %42, i64 4
+  %46 = bitcast i8 addrspace(2)* %45 to i16 addrspace(2)*
+  %47 = load i16, i16 addrspace(2)* %46, align 4
+  %48 = zext i16 %47 to i32
+  %49 = mul i32 %44, %48
+  %50 = add i32 %49, %43
+  %51 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %52 = zext i32 %50 to i64
+  %53 = bitcast i8 addrspace(2)* %51 to i64 addrspace(2)*
+  %54 = load i64, i64 addrspace(2)* %53, align 8
+  %55 = add i64 %54, %52
+  %56 = add i64 %41, %55
+  %57 = getelementptr inbounds i32, i32 addrspace(1)* %39, i64 %56
+  store i32 %38, i32 addrspace(1)* %57, align 4
+  fence syncscope("workgroup") release
+  call void @llvm.amdgcn.s.barrier()
+  fence syncscope("workgroup") acquire
+  br label %58
+
+; <label>:58:                                     ; preds = %22
+  %59 = load i32, i32* %3, align 4
+  %60 = add nsw i32 %59, 1
+  store i32 %60, i32* %3, align 4
+  br label %4
+
+; <label>:61:                                     ; preds = %4
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_global_local
+; GCN: v_mov_b32_e32 v[[VAL:[0-9]+]], 0x999
+; GCN: ds_write_b32 v{{[0-9]+}}, v[[VAL]]
+; GCN: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
+; GCN-NEXT: s_barrier
+; GCN: flat_store_dword
+define amdgpu_kernel void @test_global_local(i32 addrspace(1)*) {
+  %2 = alloca i32 addrspace(1)*, align 4
+  store i32 addrspace(1)* %0, i32 addrspace(1)** %2, align 4
+  %3 = load i32 addrspace(1)*, i32 addrspace(1)** %2, align 4
+  %4 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %5 = call i32 @llvm.amdgcn.workitem.id.x()
+  %6 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %7 = getelementptr inbounds i8, i8 addrspace(2)* %4, i64 4
+  %8 = bitcast i8 addrspace(2)* %7 to i16 addrspace(2)*
+  %9 = load i16, i16 addrspace(2)* %8, align 4
+  %10 = zext i16 %9 to i32
+  %11 = mul i32 %6, %10
+  %12 = add i32 %11, %5
+  %13 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %14 = zext i32 %12 to i64
+  %15 = bitcast i8 addrspace(2)* %13 to i64 addrspace(2)*
+  %16 = load i64, i64 addrspace(2)* %15, align 8
+  %17 = add i64 %16, %14
+  %18 = getelementptr inbounds i32, i32 addrspace(1)* %3, i64 %17
+  store i32 1, i32 addrspace(1)* %18, align 4
+  %19 = call i32 @llvm.amdgcn.workitem.id.x()
+  %20 = zext i32 %19 to i64
+  %21 = icmp eq i64 %20, 0
+  br i1 %21, label %22, label %23
+
+; <label>:22:                                     ; preds = %1
+  store i32 2457, i32 addrspace(3)* getelementptr inbounds ([1 x i32], [1 x i32] addrspace(3)* @test_global_local.temp, i64 0, i64 0), align 4
+  br label %23
+
+; <label>:23:                                     ; preds = %22, %1
+  fence syncscope("workgroup") release
+  call void @llvm.amdgcn.s.barrier()
+  fence syncscope("workgroup") acquire
+  %24 = load i32, i32 addrspace(3)* getelementptr inbounds ([1 x i32], [1 x i32] addrspace(3)* @test_global_local.temp, i64 0, i64 0), align 4
+  %25 = load i32 addrspace(1)*, i32 addrspace(1)** %2, align 4
+  %26 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %27 = call i32 @llvm.amdgcn.workitem.id.x()
+  %28 = call i32 @llvm.amdgcn.workgroup.id.x()
+  %29 = getelementptr inbounds i8, i8 addrspace(2)* %26, i64 4
+  %30 = bitcast i8 addrspace(2)* %29 to i16 addrspace(2)*
+  %31 = load i16, i16 addrspace(2)* %30, align 4
+  %32 = zext i16 %31 to i32
+  %33 = mul i32 %28, %32
+  %34 = add i32 %33, %27
+  %35 = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %36 = zext i32 %34 to i64
+  %37 = bitcast i8 addrspace(2)* %35 to i64 addrspace(2)*
+  %38 = load i64, i64 addrspace(2)* %37, align 8
+  %39 = add i64 %38, %36
+  %40 = getelementptr inbounds i32, i32 addrspace(1)* %25, i64 %39
+  store i32 %24, i32 addrspace(1)* %40, align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/fix-wwm-liveness.mir b/test/CodeGen/AMDGPU/fix-wwm-liveness.mir
new file mode 100644
index 0000000000000..101ba009cbc83
--- /dev/null
+++ b/test/CodeGen/AMDGPU/fix-wwm-liveness.mir
@@ -0,0 +1,73 @@
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-fix-wwm-liveness -o -  %s | FileCheck %s
+#CHECK: %exec = EXIT_WWM killed %19, implicit %21
+
+---
+name:            test_wwm_liveness
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:       
+  - { id: 0, class: sreg_64, preferred-register: '' }
+  - { id: 1, class: sgpr_32, preferred-register: '' }
+  - { id: 2, class: sgpr_32, preferred-register: '' }
+  - { id: 3, class: vgpr_32, preferred-register: '' }
+  - { id: 4, class: vgpr_32, preferred-register: '' }
+  - { id: 5, class: vgpr_32, preferred-register: '' }
+  - { id: 6, class: vgpr_32, preferred-register: '' }
+  - { id: 7, class: vgpr_32, preferred-register: '' }
+  - { id: 8, class: sreg_64, preferred-register: '%vcc' }
+  - { id: 9, class: sreg_64, preferred-register: '' }
+  - { id: 10, class: sreg_32_xm0, preferred-register: '' }
+  - { id: 11, class: sreg_64, preferred-register: '' }
+  - { id: 12, class: sreg_32_xm0, preferred-register: '' }
+  - { id: 13, class: sreg_32_xm0, preferred-register: '' }
+  - { id: 14, class: sreg_32_xm0, preferred-register: '' }
+  - { id: 15, class: sreg_128, preferred-register: '' }
+  - { id: 16, class: vgpr_32, preferred-register: '' }
+  - { id: 17, class: vgpr_32, preferred-register: '' }
+  - { id: 18, class: vgpr_32, preferred-register: '' }
+  - { id: 19, class: sreg_64, preferred-register: '' }
+  - { id: 20, class: sreg_64, preferred-register: '' }
+  - { id: 21, class: vgpr_32, preferred-register: '' }
+  - { id: 22, class: sreg_64, preferred-register: '' }
+  - { id: 23, class: sreg_64, preferred-register: '' }
+liveins:         
+body:             |
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  
+    %21 = V_MOV_B32_e32 0, implicit %exec
+    %5 = V_MBCNT_LO_U32_B32_e64 -1, 0, implicit %exec
+    %6 = V_MBCNT_HI_U32_B32_e32 -1, killed %5, implicit %exec
+    %8 = V_CMP_GT_U32_e64 32, killed %6, implicit %exec
+    %22 = COPY %exec, implicit-def %exec
+    %23 = S_AND_B64 %22, %8, implicit-def dead %scc
+    %0 = S_XOR_B64 %23, %22, implicit-def dead %scc
+    %exec = S_MOV_B64_term killed %23
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
+  
+  bb.1:
+    successors: %bb.2(0x80000000)
+  
+    %13 = S_MOV_B32 61440
+    %14 = S_MOV_B32 -1
+    %15 = REG_SEQUENCE undef %12, 1, undef %10, 2, killed %14, 3, killed %13, 4
+    %19 = COPY %exec
+    %exec = S_MOV_B64 -1
+    %16 = BUFFER_LOAD_DWORD_OFFSET %15, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4)
+    %17 = V_ADD_F32_e32 1065353216, killed %16, implicit %exec
+    %exec = EXIT_WWM killed %19
+    %21 = V_MOV_B32_e32 1, implicit %exec
+    early-clobber %18 = WWM killed %17, implicit %exec
+    BUFFER_STORE_DWORD_OFFSET killed %18, killed %15, 0, 0, 0, 0, 0, implicit %exec :: (store 4)
+  
+  bb.2:
+    %exec = S_OR_B64 %exec, killed %0, implicit-def %scc
+    %vgpr0 = COPY killed %21
+    SI_RETURN_TO_EPILOG killed %vgpr0
+
+...
diff --git a/test/CodeGen/AMDGPU/flat-load-clustering.mir b/test/CodeGen/AMDGPU/flat-load-clustering.mir
new file mode 100644
index 0000000000000..71547240869b5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/flat-load-clustering.mir
@@ -0,0 +1,77 @@
+# RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -run-pass machine-scheduler -o - %s | FileCheck -check-prefix=GCN %s
+
+# CGN-LABEL: name: flat_load_clustering
+# GCN:      FLAT_LOAD_DWORD
+# GCN-NEXT: FLAT_LOAD_DWORD
+--- |
+  define amdgpu_kernel void @flat_load_clustering(i32 addrspace(1)* nocapture %arg, i32 addrspace(2)* nocapture readonly %arg1) {
+  bb:
+    %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+    %idxprom = sext i32 %tid to i64
+    %gep1 = getelementptr inbounds i32, i32 addrspace(2)* %arg1, i64 %idxprom
+    %load1 = load i32, i32 addrspace(2)* %gep1, align 4
+    %gep2 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %idxprom
+    %gep34 = getelementptr inbounds i32, i32 addrspace(2)* %gep1, i64 4
+    %load2 = load i32, i32 addrspace(2)* %gep34, align 4
+    %gep4 = getelementptr inbounds i32, i32 addrspace(1)* %gep2, i64 4
+    store i32 %load1, i32 addrspace(1)* %gep2, align 4
+    store i32 %load2, i32 addrspace(1)* %gep4, align 4
+    ret void
+  }
+
+  declare i32 @llvm.amdgcn.workitem.id.x()
+
+...
+---
+name:            flat_load_clustering
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vgpr_32 }
+  - { id: 1, class: sgpr_64 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sreg_64_xexec }
+  - { id: 4, class: sreg_64_xexec }
+  - { id: 5, class: vgpr_32 }
+  - { id: 6, class: vgpr_32 }
+  - { id: 7, class: vgpr_32 }
+  - { id: 8, class: vgpr_32 }
+  - { id: 9, class: vreg_64 }
+  - { id: 10, class: vreg_64 }
+  - { id: 11, class: vgpr_32 }
+  - { id: 12, class: vreg_64 }
+  - { id: 13, class: vreg_64 }
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%0' }
+  - { reg: '%sgpr4_sgpr5', virtual-reg: '%1' }
+body:             |
+  bb.0.bb:
+    liveins: %vgpr0, %sgpr4_sgpr5
+
+    %1 = COPY %sgpr4_sgpr5
+    %0 = COPY %vgpr0
+    %3 = S_LOAD_DWORDX2_IMM %1, 0, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
+    %4 = S_LOAD_DWORDX2_IMM %1, 8, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
+    %7 = V_LSHLREV_B32_e32 2, %0, implicit %exec
+    %2 = V_MOV_B32_e32 0, implicit %exec
+    undef %12.sub0 = V_ADD_I32_e32 %4.sub0, %7, implicit-def %vcc, implicit %exec
+    %11 = COPY %4.sub1
+    %12.sub1 = V_ADDC_U32_e32 %11, %2, implicit-def dead %vcc, implicit killed %vcc, implicit %exec
+    %5 = FLAT_LOAD_DWORD %12, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4 from %ir.gep1)
+    undef %9.sub0 = V_ADD_I32_e32 %3.sub0, %7, implicit-def %vcc, implicit %exec
+    %8 = COPY %3.sub1
+    %9.sub1 = V_ADDC_U32_e32 %8, %2, implicit-def dead %vcc, implicit killed %vcc, implicit %exec
+    undef %13.sub0 = V_ADD_I32_e32 16, %12.sub0, implicit-def %vcc, implicit %exec
+    %13.sub1 = V_ADDC_U32_e32 %12.sub1, %2, implicit-def dead %vcc, implicit killed %vcc, implicit %exec
+    %6 = FLAT_LOAD_DWORD %13, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4 from %ir.gep34)
+    undef %10.sub0 = V_ADD_I32_e32 16, %9.sub0, implicit-def %vcc, implicit %exec
+    %10.sub1 = V_ADDC_U32_e32 %9.sub1, %2, implicit-def dead %vcc, implicit killed %vcc, implicit %exec
+    FLAT_STORE_DWORD %9, %5, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4 into %ir.gep2)
+    FLAT_STORE_DWORD %10, %6, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4 into %ir.gep4)
+    S_ENDPGM
+
+...
diff --git a/test/CodeGen/AMDGPU/flat_atomics.ll b/test/CodeGen/AMDGPU/flat_atomics.ll
index 8e153181decb7..e3d307ea1ce6c 100644
--- a/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -998,8 +998,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i32_offset:
-; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
-; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16 glc{{$}}
+; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16{{$}}
 define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, i32 addrspace(4)* %out) {
 entry:
   %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
@@ -1008,7 +1008,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i32:
-; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
 define amdgpu_kernel void @atomic_store_i32(i32 %in, i32 addrspace(4)* %out) {
 entry:
   store atomic i32 %in, i32 addrspace(4)* %out seq_cst, align 4
@@ -1016,8 +1016,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i32_addr64_offset:
-; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
-; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16 glc{{$}}
+; CIVI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+; GFX9: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} offset:16{{$}}
 define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, i32 addrspace(4)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index
@@ -1027,7 +1027,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i32_addr64:
-; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
 define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, i32 addrspace(4)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(4)* %out, i64 %index
diff --git a/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index 723dde9ab68fd..acbfe6ab4cb04 100644
--- a/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -846,7 +846,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i64_offset:
-; GCN: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
 define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, i64 addrspace(4)* %out) {
 entry:
   %gep = getelementptr i64, i64 addrspace(4)* %out, i64 4
@@ -855,7 +855,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i64:
-; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] glc
+; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}]
 define amdgpu_kernel void @atomic_store_i64(i64 %in, i64 addrspace(4)* %out) {
 entry:
   store atomic i64 %in, i64 addrspace(4)* %out seq_cst, align 8
@@ -863,7 +863,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i64_addr64_offset:
-; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}}
+; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}}
 define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, i64 addrspace(4)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index
@@ -873,7 +873,7 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_store_i64_addr64:
-; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}}
+; GCN: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}}
 define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, i64 addrspace(4)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(4)* %out, i64 %index
diff --git a/test/CodeGen/AMDGPU/fma-combine.ll b/test/CodeGen/AMDGPU/fma-combine.ll
index 7526d08bdbe52..3b3c87f17f72b 100644
--- a/test/CodeGen/AMDGPU/fma-combine.ll
+++ b/test/CodeGen/AMDGPU/fma-combine.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math -mattr=+fp32-denormals < %s | FileCheck -check-prefix=SI-FMA -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math -mattr=+fp32-denormals < %s | FileCheck -enable-var-scope -check-prefix=SI-FMA -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
 
 ; Note: The SI-FMA conversions of type x * (y + 1) --> x * y + x would be
 ; beneficial even without fp32 denormals, but they do require no-infs-fp-math
@@ -237,6 +237,7 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64(double addrspace(1)* noalia
 ; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
 ; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
+; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24{{$}}
 ; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[C]]
 ; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[D]]
 ; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
@@ -271,6 +272,7 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_neg(double addrspace(
 ; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
 ; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16{{$}}
+; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24{{$}}
 ; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[C]]
 ; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], -[[D]]
 ; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
@@ -625,5 +627,51 @@ define amdgpu_kernel void @test_f64_interp(double addrspace(1)* %out,
   ret void
 }
 
+; Make sure negative constant cancels out fneg
+; GCN-LABEL: {{^}}fma_neg_2.0_neg_a_b_f32:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN-NOT: [[A]]
+; GCN-NOT: [[B]]
+; GCN: v_fma_f32 v{{[0-9]+}}, [[A]], 2.0, [[B]]
+define amdgpu_kernel void @fma_neg_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
+  %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
+  %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
+
+  %r1 = load volatile float, float addrspace(1)* %gep.0
+  %r2 = load volatile float, float addrspace(1)* %gep.1
+
+  %r1.fneg = fsub float -0.000000e+00, %r1
+
+  %r3 = tail call float @llvm.fma.f32(float -2.0, float %r1.fneg, float %r2)
+  store float %r3, float addrspace(1)* %gep.out
+  ret void
+}
+
+; GCN-LABEL: {{^}}fma_2.0_neg_a_b_f32:
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN-NOT: [[A]]
+; GCN-NOT: [[B]]
+; GCN: v_fma_f32 v{{[0-9]+}}, [[A]], -2.0, [[B]]
+define amdgpu_kernel void @fma_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid
+  %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
+  %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid
+
+  %r1 = load volatile float, float addrspace(1)* %gep.0
+  %r2 = load volatile float, float addrspace(1)* %gep.1
+
+  %r1.fneg = fsub float -0.000000e+00, %r1
+
+  %r3 = tail call float @llvm.fma.f32(float 2.0, float %r1.fneg, float %r2)
+  store float %r3, float addrspace(1)* %gep.out
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
+
diff --git a/test/CodeGen/AMDGPU/fma.ll b/test/CodeGen/AMDGPU/fma.ll
index 6be4c450a51ed..8e51f82112ff8 100644
--- a/test/CodeGen/AMDGPU/fma.ll
+++ b/test/CodeGen/AMDGPU/fma.ll
@@ -1,5 +1,12 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cedar -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=juniper -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=sumo -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=barts -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=caicos -verify-machineinstrs < %s
+; RUN:  not llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=turks -verify-machineinstrs < %s
 
 declare float @llvm.fma.f32(float, float, float) nounwind readnone
 declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) nounwind readnone
diff --git a/test/CodeGen/AMDGPU/fmax3.f64.ll b/test/CodeGen/AMDGPU/fmax3.f64.ll
index 8b9104b79e7f1..a56a5866aadd4 100644
--- a/test/CodeGen/AMDGPU/fmax3.f64.ll
+++ b/test/CodeGen/AMDGPU/fmax3.f64.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 declare double @llvm.maxnum.f64(double, double) nounwind readnone
diff --git a/test/CodeGen/AMDGPU/fmed3.ll b/test/CodeGen/AMDGPU/fmed3.ll
index 4cfc9fc80fb07..934d20716c405 100644
--- a/test/CodeGen/AMDGPU/fmed3.ll
+++ b/test/CodeGen/AMDGPU/fmed3.ll
@@ -1,9 +1,9 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -check-prefix=SNAN -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -check-prefix=SNAN -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -check-prefix=SNAN -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SNAN -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SNAN -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=NOSNAN -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp-exceptions -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SNAN -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
 
 
 ; GCN-LABEL: {{^}}v_test_nnan_input_fmed3_r_i_i_f32:
@@ -165,9 +165,9 @@ define amdgpu_kernel void @v_test_legacy_fmed3_r_i_i_f32(float addrspace(1)* %ou
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_srcmod0:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer_|flat_|global_}}load_dword [[A:v[0-9]+]]
+; GCN: {{buffer_|flat_|global_}}load_dword [[B:v[0-9]+]]
+; GCN: {{buffer_|flat_|global_}}load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, -[[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod0(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -188,9 +188,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod0(float addrs
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_srcmod1:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], -[[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod1(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -211,9 +211,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod1(float addrs
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_srcmod2:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], -[[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod2(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -234,9 +234,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod2(float addrs
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_srcmod012:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, -[[A]], |[[B]]|, -|[[C]]|
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod012(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -263,9 +263,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod012(float add
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_negabs012:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, -|[[A]]|, -|[[B]]|, -|[[C]]|
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_negabs012(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -294,9 +294,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_negabs012(float add
 }
 
 ; GCN-LABEL: {{^}}v_nnan_inputs_med3_f32_pat0:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN-DAG: v_add_f32_e32 [[A_ADD:v[0-9]+]], 1.0, [[A]]
 ; GCN-DAG: v_add_f32_e32 [[B_ADD:v[0-9]+]], 2.0, [[B]]
 ; GCN-DAG: v_add_f32_e32 [[C_ADD:v[0-9]+]], 4.0, [[C]]
@@ -337,9 +337,9 @@ define amdgpu_kernel void @v_nnan_inputs_med3_f32_pat0(float addrspace(1)* %out,
 ; + commute outermost max
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -359,9 +359,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat1:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat1(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -381,9 +381,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat1(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat2:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat2(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -403,9 +403,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat2(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat3:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat3(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -425,9 +425,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat3(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat4:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat4(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -447,9 +447,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat4(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat5:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat5(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -469,9 +469,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat5(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat6:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat6(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -491,9 +491,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat6(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat7:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat7(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -513,9 +513,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat7(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat8:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat8(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -535,9 +535,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat8(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat9:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat9(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -557,9 +557,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat9(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat10:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat10(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -579,9 +579,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat10(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat11:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat11(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -601,9 +601,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat11(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat12:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat12(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -623,9 +623,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat12(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat13:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat13(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -645,9 +645,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat13(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat14:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[A]], [[B]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat14(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -667,9 +667,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat14(float addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat15:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_med3_f32 v{{[0-9]+}}, [[B]], [[A]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat15(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -842,9 +842,9 @@ define amdgpu_kernel void @v_nnan_inputs_missing2_med3_f32_pat0(float addrspace(
 }
 
 ; GCN-LABEL: {{^}}v_test_global_nnans_med3_f32_pat0_srcmod0_mismatch:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN-DAG: v_min_f32
 ; GCN-DAG: v_max_f32
 ; GCN-DAG: v_min_f32
@@ -869,9 +869,9 @@ define amdgpu_kernel void @v_test_global_nnans_med3_f32_pat0_srcmod0_mismatch(fl
 
 ; A simple min and max is not sufficient
 ; GCN-LABEL: {{^}}v_test_global_nnans_min_max_f32:
-; GCN: {{buffer_|flat_}}load_dword [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_dword [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[C:v[0-9]+]]
 ; GCN: v_max_f32_e32 [[MAX:v[0-9]+]], [[A]], [[B]]
 ; GCN: v_min_f32_e32 v{{[0-9]+}}, [[MAX]], [[C]]
 define amdgpu_kernel void @v_test_global_nnans_min_max_f32(float addrspace(1)* %out, float addrspace(1)* %aptr, float addrspace(1)* %bptr, float addrspace(1)* %cptr) #2 {
@@ -899,7 +899,7 @@ define amdgpu_kernel void @v_test_global_nnans_min_max_f32(float addrspace(1)* %
 ; VI: v_max_f16_e32 v{{[0-9]+}}, 2.0
 ; VI: v_min_f16_e32 v{{[0-9]+}}, 4.0
 
-; GFX9: v_add_f16_e32 v{{[0-9]+}}, 1.0
+; GFX9: v_add_f16_e32 [[ADD:v[0-9]+]], 1.0
 ; GFX9: v_med3_f16 v{{[0-9]+}}, [[ADD]], 2.0, 4.0
 define amdgpu_kernel void @v_test_nnan_input_fmed3_r_i_i_f16(half addrspace(1)* %out, half addrspace(1)* %aptr) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -915,9 +915,9 @@ define amdgpu_kernel void @v_test_nnan_input_fmed3_r_i_i_f16(half addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_nnan_inputs_med3_f16_pat0:
-; GCN: {{buffer_|flat_}}load_ushort [[A:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_ushort [[B:v[0-9]+]]
-; GCN: {{buffer_|flat_}}load_ushort [[C:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[C:v[0-9]+]]
 
 ; SI: v_cvt_f32_f16
 ; SI: v_cvt_f32_f16
diff --git a/test/CodeGen/AMDGPU/fminnum.f64.ll b/test/CodeGen/AMDGPU/fminnum.f64.ll
index 01b267411212c..475615e52cb93 100644
--- a/test/CodeGen/AMDGPU/fminnum.f64.ll
+++ b/test/CodeGen/AMDGPU/fminnum.f64.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 declare double @llvm.minnum.f64(double, double) #0
diff --git a/test/CodeGen/AMDGPU/fmul.f16.ll b/test/CodeGen/AMDGPU/fmul.f16.ll
index 5f120f63d7fe3..fb8aeb80e152a 100644
--- a/test/CodeGen/AMDGPU/fmul.f16.ll
+++ b/test/CodeGen/AMDGPU/fmul.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}fmul_f16
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/fmuladd.f16.ll b/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 980d68ceded87..0fdb9453c5869 100644
--- a/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -1,12 +1,12 @@
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s
 
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare half @llvm.fmuladd.f16(half, half, half) #1
@@ -154,7 +154,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_a_b_f16(half addrspace(1)* %out, half
 ; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]]
 ; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]]
 
-; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], -[[R1]], -2.0, [[R2]]
+; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]]
 ; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -178,7 +178,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f16(half addrspace(1)* %out,
 ; VI-FLUSH: v_mac_f16_e32 [[R2]], -2.0, [[R1]]
 ; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]]
 
-; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], -[[R1]], 2.0, [[R2]]
+; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], -2.0, [[R2]]
 ; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @fmuladd_2.0_neg_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -441,7 +441,7 @@ define amdgpu_kernel void @fsub_c_fadd_a_a_f16(half addrspace(1)* %out, half add
 
 ; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
-; VI-DENORM-CONTRACT: v_fma_f16 [[R2]], [[R1]], 2.0, -[[R2]]
+; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]]
 
 ; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; VI-DENORM-STRICT: v_sub_f16_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]]
diff --git a/test/CodeGen/AMDGPU/fmuladd.f32.ll b/test/CodeGen/AMDGPU/fmuladd.f32.ll
index 4b1e41ff91e17..fee3c95c47378 100644
--- a/test/CodeGen/AMDGPU/fmuladd.f32.ll
+++ b/test/CodeGen/AMDGPU/fmuladd.f32.ll
@@ -218,7 +218,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_a_b_f32(float addrspace(1)* %out, flo
 ; SI-FLUSH: buffer_store_dword [[R2]]
 ; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]]
 
-; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], -[[R1]], -2.0, [[R2]]
+; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]]
 
 ; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]
@@ -249,7 +249,7 @@ define amdgpu_kernel void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out,
 ; SI-FLUSH: buffer_store_dword [[R2]]
 ; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]]
 
-; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], -[[R1]], 2.0, [[R2]]
+; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], -2.0, [[R2]]
 
 ; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]]
 ; GCN-DENORM-SLOWFMA: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]]
diff --git a/test/CodeGen/AMDGPU/fmuladd.v2f16.ll b/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
index b50a26c023ca3..4a9beef858402 100644
--- a/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
+++ b/test/CodeGen/AMDGPU/fmuladd.v2f16.ll
@@ -1,12 +1,12 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-FLUSH,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-FLUSH,GFX9 %s
 
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=on -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,GFX9-DENORM-STRICT,GFX9-DENORM,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=+fp64-fp16-denormals -fp-contract=fast -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,GFX9-DENORM-CONTRACT,GFX9-DENORM,GFX9 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare <2 x half> @llvm.fmuladd.v2f16(<2 x half>, <2 x half>, <2 x half>) #1
@@ -28,15 +28,15 @@ define amdgpu_kernel void @fmuladd_v2f16(<2 x half> addrspace(1)* %out, <2 x hal
 }
 
 ; GCN-LABEL: {{^}}fmuladd_2.0_a_b_v2f16:
-; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]],
-; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R1:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R2:v[0-9]+]],
 ; GFX9-FLUSH: v_pk_add_f16 [[ADD0:v[0-9]+]], [[R1]], [[R1]]
 ; GFX9-FLUSH: v_pk_add_f16 [[RESULT:v[0-9]+]], [[ADD0]], [[R2]]
 
-; GFX9-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX9-FLUSH: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 
 ; GFX9-DENORM: v_pk_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]]
-; GFX9-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX9-DENORM: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @fmuladd_2.0_a_b_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
@@ -52,15 +52,15 @@ define amdgpu_kernel void @fmuladd_2.0_a_b_v2f16(<2 x half> addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}fmuladd_a_2.0_b_v2f16:
-; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]],
-; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R1:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R2:v[0-9]+]],
 ; GFX9-FLUSH: v_pk_add_f16 [[ADD0:v[0-9]+]], [[R1]], [[R1]]
 ; GFX9-FLUSH: v_pk_add_f16 [[RESULT:v[0-9]+]], [[ADD0]], [[R2]]
 
-; GFX9-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX9-FLUSH: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 
 ; GFX9-DENORM: v_pk_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]]
-; GFX9-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GFX9-DENORM: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @fmuladd_a_2.0_b_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr <2 x half>, <2 x half> addrspace(1)* %out, i32 %tid
@@ -76,8 +76,8 @@ define amdgpu_kernel void @fmuladd_a_2.0_b_v2f16(<2 x half> addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}fadd_a_a_b_v2f16:
-; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]],
-; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R1:v[0-9]+]],
+; GCN: {{buffer|flat|global}}_load_dword [[R2:v[0-9]+]],
 ; GFX9-FLUSH: v_pk_add_f16 [[ADD0:v[0-9]+]], [[R1]], [[R1]]
 ; GFX9-FLUSH: v_pk_add_f16 [[RESULT:v[0-9]+]], [[ADD0]], [[R2]]
 
@@ -85,7 +85,7 @@ define amdgpu_kernel void @fmuladd_a_2.0_b_v2f16(<2 x half> addrspace(1)* %out,
 ; GFX9-DENORM-STRICT: v_pk_add_f16 [[RESULT:v[0-9]+]], [[ADD0]], [[R2]]
 
 ; GFX9-DENORM-CONTRACT: v_pk_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @fadd_a_a_b_v2f16(<2 x half> addrspace(1)* %out,
                             <2 x half> addrspace(1)* %in1,
                             <2 x half> addrspace(1)* %in2) #0 {
diff --git a/test/CodeGen/AMDGPU/fneg-combines.ll b/test/CodeGen/AMDGPU/fneg-combines.ll
index 002bc47fb96ae..fd3d4f053e957 100644
--- a/test/CodeGen/AMDGPU/fneg-combines.ll
+++ b/test/CodeGen/AMDGPU/fneg-combines.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -start-after=sink -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -enable-no-signed-zeros-fp-math -march=amdgcn -mcpu=tahiti -start-after=sink -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NSZ -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -start-after=sink -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-SAFE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -enable-no-signed-zeros-fp-math -march=amdgcn -mcpu=tahiti -start-after=sink -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-NSZ -check-prefix=SI -check-prefix=FUNC %s
 
 ; --------------------------------------------------------------------------------
 ; fadd tests
@@ -59,7 +59,7 @@ define amdgpu_kernel void @v_fneg_add_store_use_add_f32(float addrspace(1)* %out
 ; GCN-SAFE: v_mul_f32_e32 [[MUL:v[0-9]+]], 4.0, [[ADD]]
 
 ; GCN-NSZ: v_sub_f32_e64 [[NEG_ADD:v[0-9]+]], -[[A]], [[B]]
-; GCN-NSZ-NEXT: v_mul_f32_e32 [[MUL:v[0-9]+]], -4.0, [[ADD]]
+; GCN-NSZ-NEXT: v_mul_f32_e32 [[MUL:v[0-9]+]], -4.0, [[NEG_ADD]]
 ; GCN: buffer_store_dword [[NEG_ADD]]
 ; GCN-NEXT: buffer_store_dword [[MUL]]
 define amdgpu_kernel void @v_fneg_add_multi_use_add_f32(float addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
@@ -513,7 +513,7 @@ define amdgpu_kernel void @v_fneg_0_minnum_foldable_use_f32(float addrspace(1)*
 ; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_max_f32_e64 [[MAX0:v[0-9]+]], -[[A]], -[[B]]
-; GCN-NEXT: v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MUL0]]
+; GCN-NEXT: v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MAX0]]
 ; GCN-NEXT: buffer_store_dword [[MAX0]]
 ; GCN-NEXT: buffer_store_dword [[MUL1]]
 define amdgpu_kernel void @v_fneg_minnum_multi_use_minnum_f32(float addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
@@ -660,7 +660,7 @@ define amdgpu_kernel void @v_fneg_0_maxnum_foldable_use_f32(float addrspace(1)*
 ; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_min_f32_e64 [[MAX0:v[0-9]+]], -[[A]], -[[B]]
-; GCN-NEXT: v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MUL0]]
+; GCN-NEXT: v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MAX0]]
 ; GCN-NEXT: buffer_store_dword [[MAX0]]
 ; GCN-NEXT: buffer_store_dword [[MUL1]]
 define amdgpu_kernel void @v_fneg_maxnum_multi_use_maxnum_f32(float addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
diff --git a/test/CodeGen/AMDGPU/fneg-fabs.f16.ll b/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
index 56aea641d16e6..4d502b1af87f3 100644
--- a/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
+++ b/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=CIVI %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GFX89 -check-prefix=GCN -check-prefix=CIVI %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx901 -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX89 -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX89 -check-prefix=GFX9 -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}fneg_fabs_fadd_f16:
 ; CI: v_cvt_f32_f16_e32
@@ -26,7 +26,7 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(half addrspace(1)* %out, half %x,
 ; GFX89-NOT: _and
 ; GFX89: v_mul_f16_e64 [[MUL:v[0-9]+]], {{v[0-9]+}}, -|{{v[0-9]+}}|
 ; GFX89-NOT: [[MUL]]
-; GFX89: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[MUL]]
+; GFX89: {{flat|global}}_store_short v{{\[[0-9]+:[0-9]+\]}}, [[MUL]]
 define amdgpu_kernel void @fneg_fabs_fmul_f16(half addrspace(1)* %out, half %x, half %y) {
   %fabs = call half @llvm.fabs.f16(half %x)
   %fsub = fsub half -0.0, %fabs
@@ -70,11 +70,13 @@ define amdgpu_kernel void @v_fneg_fabs_f16(half addrspace(1)* %out, half addrspa
 
 ; FIXME: single bit op
 ; GCN-LABEL: {{^}}s_fneg_fabs_v2f16:
-; CIVI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}}
+; CI: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 16, v{{[0-9]+}}
+; CI: v_or_b32_e32 [[OR:v[0-9]+]], v{{[0-9]+}}, [[SHL]]
+; CI: v_or_b32_e32 v{{[0-9]+}}, 0x80008000, [[OR]]
+; VI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}}
 ; VI: v_mov_b32_e32 [[VMASK:v[0-9]+]], [[MASK]]
-; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, [[VMASK]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; CIVI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
+; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
 ; CIVI: flat_store_dword
 
 ; GFX9: s_or_b32 s{{[0-9]+}}, 0x80008000, s{{[0-9]+}}
@@ -86,11 +88,14 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x
 }
 
 ; GCN-LABEL: {{^}}fneg_fabs_v4f16:
-; CIVI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}}
-; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
-; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
-; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
-; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
+; CI: s_mov_b32 [[MASK:s[0-9]+]], 0x80008000
+; CI: v_lshlrev_b32_e32 [[SHL0:v[0-9]+]], 16, v{{[0-9]+}}
+; CI: v_or_b32_e32 [[OR0:v[0-9]+]], v{{[0-9]+}}, [[SHL0]]
+; CI: v_lshlrev_b32_e32 [[SHL1:v[0-9]+]], 16, v{{[0-9]+}}
+; CI: v_or_b32_e32 [[OR1:v[0-9]+]], v{{[0-9]+}}, [[SHL1]]
+; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], [[OR0]]
+; CI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], [[OR1]]
+; VI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}}
 ; VI: v_mov_b32_e32 [[VMASK:v[0-9]+]], [[MASK]]
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, [[VMASK]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]],
@@ -101,7 +106,7 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x
 ; GFX9: s_or_b32 s{{[0-9]+}}, [[MASK]], s{{[0-9]+}}
 ; GFX9: s_or_b32 s{{[0-9]+}}, [[MASK]], s{{[0-9]+}}
 
-; GCN: flat_store_dwordx2
+; GCN: {{flat|global}}_store_dwordx2
 define amdgpu_kernel void @fneg_fabs_v4f16(<4 x half> addrspace(1)* %out, <4 x half> %in) {
   %fabs = call <4 x half> @llvm.fabs.v4f16(<4 x half> %in)
   %fsub = fsub <4 x half> <half -0.0, half -0.0, half -0.0, half -0.0>, %fabs
diff --git a/test/CodeGen/AMDGPU/fneg.f16.ll b/test/CodeGen/AMDGPU/fneg.f16.ll
index 49d6742527468..5649ddfc6e39e 100644
--- a/test/CodeGen/AMDGPU/fneg.f16.ll
+++ b/test/CodeGen/AMDGPU/fneg.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=kaveri -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=CIVI -check-prefix=GCN %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=CIVI -check-prefix=GCN -check-prefix=GFX89 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN -check-prefix=GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=kaveri -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI,GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
 
 ; FIXME: Should be able to do scalar op
 ; GCN-LABEL: {{^}}s_fneg_f16:
@@ -14,7 +14,7 @@ define amdgpu_kernel void @s_fneg_f16(half addrspace(1)* %out, half %in) #0 {
 ; well.
 
 ; GCN-LABEL: {{^}}v_fneg_f16:
-; GCN: flat_load_ushort [[VAL:v[0-9]+]],
+; GCN: {{flat|global}}_load_ushort [[VAL:v[0-9]+]],
 ; GCN: v_xor_b32_e32 [[XOR:v[0-9]+]], 0x8000, [[VAL]]
 ; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[XOR]]
 ; SI: buffer_store_short [[XOR]]
@@ -29,7 +29,7 @@ define amdgpu_kernel void @v_fneg_f16(half addrspace(1)* %out, half addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}fneg_free_f16:
-; GCN: flat_load_ushort [[NEG_VALUE:v[0-9]+]],
+; GCN: {{flat|global}}_load_ushort [[NEG_VALUE:v[0-9]+]],
 
 ; XCI: s_xor_b32 [[XOR:s[0-9]+]], [[NEG_VALUE]], 0x8000{{$}}
 ; CI: v_xor_b32_e32 [[XOR:v[0-9]+]], 0x8000, [[NEG_VALUE]]
@@ -42,7 +42,7 @@ define amdgpu_kernel void @fneg_free_f16(half addrspace(1)* %out, i16 %in) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_fneg_fold_f16:
-; GCN: flat_load_ushort [[NEG_VALUE:v[0-9]+]]
+; GCN: {{flat|global}}_load_ushort [[NEG_VALUE:v[0-9]+]]
 
 ; CI-DAG: v_cvt_f32_f16_e32 [[CVT_VAL:v[0-9]+]], [[NEG_VALUE]]
 ; CI-DAG: v_cvt_f32_f16_e64 [[NEG_CVT0:v[0-9]+]], -[[NEG_VALUE]]
@@ -81,7 +81,7 @@ define amdgpu_kernel void @s_fneg_v2f16(<2 x half> addrspace(1)* %out, <2 x half
 }
 
 ; GCN-LABEL: {{^}}v_fneg_v2f16:
-; GCN: flat_load_dword [[VAL:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80008000, [[VAL]]
 define amdgpu_kernel void @v_fneg_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -107,7 +107,7 @@ define amdgpu_kernel void @fneg_free_v2f16(<2 x half> addrspace(1)* %out, i32 %i
 }
 
 ; GCN-LABEL: {{^}}v_fneg_fold_v2f16:
-; GCN: flat_load_dword [[VAL:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 
 ; CI: v_cvt_f32_f16_e64 v{{[0-9]+}}, -v{{[0-9]+}}
 ; CI: v_cvt_f32_f16_e64 v{{[0-9]+}}, -v{{[0-9]+}}
@@ -116,8 +116,7 @@ define amdgpu_kernel void @fneg_free_v2f16(<2 x half> addrspace(1)* %out, i32 %i
 ; CI: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CI: v_cvt_f16_f32
 
-; VI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16,
-; VI: v_mul_f16_sdwa v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI: v_mul_f16_sdwa v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 ; VI: v_mul_f16_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
 
 ; GFX9: v_pk_mul_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} neg_lo:[1,0] neg_hi:[1,0]{{$}}
@@ -130,7 +129,7 @@ define amdgpu_kernel void @v_fneg_fold_v2f16(<2 x half> addrspace(1)* %out, <2 x
 }
 
 ; GCN-LABEL: {{^}}v_extract_fneg_fold_v2f16:
-; GCN-DAG: flat_load_dword [[VAL:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 ; CI-DAG: v_mul_f32_e32 v{{[0-9]+}}, -4.0, v{{[0-9]+}}
 ; CI-DAG: v_sub_f32_e32 v{{[0-9]+}}, 2.0, v{{[0-9]+}}
 
@@ -152,9 +151,10 @@ define amdgpu_kernel void @v_extract_fneg_fold_v2f16(<2 x half> addrspace(1)* %i
 }
 
 ; GCN-LABEL: {{^}}v_extract_fneg_no_fold_v2f16:
-; GCN: flat_load_dword [[VAL:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VAL:v[0-9]+]]
 ; GCN: v_xor_b32_e32 [[NEG:v[0-9]+]], 0x80008000, [[VAL]]
-; GCN: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[NEG]]
+; CIVI: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[NEG]]
+; GFX9: global_store_short_d16_hi v{{\[[0-9]+:[0-9]+\]}}, [[NEG]], off
 define amdgpu_kernel void @v_extract_fneg_no_fold_v2f16(<2 x half> addrspace(1)* %in) #0 {
   %val = load <2 x half>, <2 x half> addrspace(1)* %in
   %fneg = fsub <2 x half> <half -0.0, half -0.0>, %val
diff --git a/test/CodeGen/AMDGPU/fneg.ll b/test/CodeGen/AMDGPU/fneg.ll
index d1eabfb13c9af..94ec61622bd26 100644
--- a/test/CodeGen/AMDGPU/fneg.ll
+++ b/test/CodeGen/AMDGPU/fneg.ll
@@ -84,3 +84,15 @@ define amdgpu_kernel void @fneg_fold_f32(float addrspace(1)* %out, float %in) {
   store float %fmul, float addrspace(1)* %out
   ret void
 }
+
+; Make sure we turn some integer operations back into fabs
+; FUNC-LABEL: {{^}}bitpreserve_fneg_f32:
+; GCN: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -4.0
+define amdgpu_kernel void @bitpreserve_fneg_f32(float addrspace(1)* %out, float %in) {
+  %in.bc = bitcast float %in to i32
+  %int.abs = xor i32 %in.bc, 2147483648
+  %bc = bitcast i32 %int.abs to float
+  %fadd = fmul float %bc, 4.0
+  store float %fadd, float addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/fold-cndmask.mir b/test/CodeGen/AMDGPU/fold-cndmask.mir
index 8dfec91663038..1ddb02a59b962 100644
--- a/test/CodeGen/AMDGPU/fold-cndmask.mir
+++ b/test/CodeGen/AMDGPU/fold-cndmask.mir
@@ -1,11 +1,11 @@
 # RUN: llc -march=amdgcn -run-pass si-fold-operands -verify-machineinstrs -o - %s | FileCheck %s
 
-# CHECK: %1 = V_MOV_B32_e32 0, implicit %exec
-# CHECK: %2 = V_MOV_B32_e32 0, implicit %exec
-# CHECK: %4 = COPY %3
-# CHECK: %5 = V_MOV_B32_e32 0, implicit %exec
-# CHECK: %6 = V_MOV_B32_e32 0, implicit %exec
-# CHECK: %7 = COPY %3
+# CHECK: %1:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# CHECK: %2:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# CHECK: %4:vgpr_32 = COPY %3
+# CHECK: %5:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# CHECK: %6:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# CHECK: %7:vgpr_32 = COPY %3
 
 ---
 name:            fold_cndmask
diff --git a/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir b/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
index 3155b7a8664fb..9831538aa66a3 100644
--- a/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
+++ b/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
@@ -1,8 +1,8 @@
 # RUN: llc -march=amdgcn -run-pass peephole-opt -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
 ...
 # GCN-LABEL: name: no_fold_imm_madak_mac_clamp_f32
-# GCN: %23 = V_MOV_B32_e32 1090519040, implicit %exec
-# GCN-NEXT: %24 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 1090519040, implicit %exec
+# GCN-NEXT: %24:vgpr_32 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit %exec
 
 name:            no_fold_imm_madak_mac_clamp_f32
 tracksRegLiveness: true
@@ -72,8 +72,8 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: no_fold_imm_madak_mac_omod_f32
-# GCN: %23 = V_MOV_B32_e32 1090519040, implicit %exec
-# GCN: %24 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 0, 2, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 1090519040, implicit %exec
+# GCN: %24:vgpr_32 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 0, 2, implicit %exec
 
 name:            no_fold_imm_madak_mac_omod_f32
 tracksRegLiveness: true
@@ -143,8 +143,8 @@ body:             |
 ...
 ---
 # GCN: name: no_fold_imm_madak_mad_clamp_f32
-# GCN: %23 = V_MOV_B32_e32 1090519040, implicit %exec
-# GCN: %24 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 1090519040, implicit %exec
+# GCN: %24:vgpr_32 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit %exec
 
 name:            no_fold_imm_madak_mad_clamp_f32
 tracksRegLiveness: true
@@ -214,8 +214,8 @@ body:             |
 ...
 ---
 # GCN: name: no_fold_imm_madak_mad_omod_f32
-# GCN: %23 = V_MOV_B32_e32 1090519040, implicit %exec
-# GCN: %24 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 0, 1, implicit %exec
+# GCN: %23:vgpr_32 = V_MOV_B32_e32 1090519040, implicit %exec
+# GCN: %24:vgpr_32 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 0, 1, implicit %exec
 
 name:            no_fold_imm_madak_mad_omod_f32
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/fold-operands-order.mir b/test/CodeGen/AMDGPU/fold-operands-order.mir
index 51bb357fcf6ee..3f28f39930f57 100644
--- a/test/CodeGen/AMDGPU/fold-operands-order.mir
+++ b/test/CodeGen/AMDGPU/fold-operands-order.mir
@@ -6,10 +6,10 @@
 # aren't made in users before the def is seen.
 
 # GCN-LABEL: name: mov_in_use_list_2x{{$}}
-# GCN: %2 = V_MOV_B32_e32 0, implicit %exec
-# GCN-NEXT: %3 = COPY undef %0
+# GCN: %2:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# GCN-NEXT: %3:vgpr_32 = COPY undef %0
 
-# GCN: %1 = V_MOV_B32_e32 0, implicit %exec
+# GCN: %1:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
 
 
 name: mov_in_use_list_2x
diff --git a/test/CodeGen/AMDGPU/fpext-free.ll b/test/CodeGen/AMDGPU/fpext-free.ll
new file mode 100644
index 0000000000000..0a504b3e03e4e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/fpext-free.ll
@@ -0,0 +1,384 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9,GFX9-F32DENORM %s
+; RUN: llc -march=amdgcn -mcpu=gfx803 -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32FLUSH %s
+; RUN: llc -march=amdgcn -mcpu=gfx803 -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,VI,VI-F32DENORM %s
+
+;  fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
+
+; GCN-LABEL: {{^}}fadd_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16
+; GFX9-F32DENORM-NEXT: v_add_f32
+define float @fadd_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %mul.ext = fpext half %mul to float
+  %add = fadd float %mul.ext, %z
+  ret float %add
+}
+
+; f16->f64 is not free.
+; GCN-LABEL: {{^}}fadd_fpext_fmul_f16_to_f64:
+; GFX89: v_mul_f16
+; GFX89: v_cvt_f32_f16
+; GFX89: v_cvt_f64_f32
+; GFX89: v_add_f64
+define double @fadd_fpext_fmul_f16_to_f64(half %x, half %y, double %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %mul.ext = fpext half %mul to double
+  %add = fadd double %mul.ext, %z
+  ret double %add
+}
+
+; f32->f64 is not free.
+; GCN-LABEL: {{^}}fadd_fpext_fmul_f32_to_f64:
+; GCN: v_mul_f32
+; GCN: v_cvt_f64_f32
+; GCN: v_add_f64
+define double @fadd_fpext_fmul_f32_to_f64(float %x, float %y, double %z) #0 {
+entry:
+  %mul = fmul float %x, %y
+  %mul.ext = fpext float %mul to double
+  %add = fadd double %mul.ext, %z
+  ret double %add
+}
+
+; fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
+; GCN-LABEL: {{^}}fadd_fpext_fmul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16
+; GFX9-F32DENORM-NEXT: v_add_f32
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fadd_fpext_fmul_f16_to_f32_commute(half %x, half %y, float %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %mul.ext = fpext half %mul to float
+  %add = fadd float %z, %mul.ext
+  ret float %add
+}
+
+; fold (fadd (fma x, y, (fpext (fmul u, v))), z)
+;   -> (fma x, y, (fma (fpext u), (fpext v), z))
+
+; GCN-LABEL: {{^}}fadd_muladd_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16
+; GFX9-F32DENORM-NEXT: v_fma_f32
+; GFX9-F32DENORM-NEXT: v_add_f32
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fadd_muladd_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
+  %add = fadd float %fma, %z
+  ret float %add
+}
+
+; fold (fadd x, (fma y, z, (fpext (fmul u, v)))
+;   -> (fma y, z, (fma (fpext u), (fpext v), x))
+; GCN-LABEL: {{^}}fadd_muladd_fpext_fmul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16
+; GFX9-F32DENORM-NEXT: v_fma_f32
+; GFX9-F32DENORM-NEXT: v_add_f32
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fadd_muladd_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
+  %add = fadd float %z, %fma
+  ret float %add
+}
+
+; GCN-LABEL: {{^}}fadd_fmad_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v2, v3, v4 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v2, v2, v3
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v2
+define float @fadd_fmad_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %mul.ext = fpext half %mul to float
+  %mul1 = fmul contract float %x, %y
+  %fmad = fadd contract float %mul1, %mul.ext
+  %add = fadd float %fmad, %z
+  ret float %add
+}
+
+; fold (fadd (fma x, y, (fpext (fmul u, v))), z)
+;   -> (fma x, y, (fma (fpext u), (fpext v), z))
+
+; GCN-LABEL: {{^}}fadd_fma_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX89: v_mul_f16
+; GFX89: v_cvt_f32_f16
+; GFX89: v_fma_f32
+; GFX89: v_add_f32
+define float @fadd_fma_fpext_fmul_f16_to_f32(float %x, float %y, half %u, half %v, float %z) #0 {
+entry:
+  %mul = fmul contract half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fma.f32(float %x, float %y, float %mul.ext)
+  %add = fadd float %fma, %z
+  ret float %add
+}
+
+; GCN-LABEL: {{^}}fadd_fma_fpext_fmul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX89: v_mul_f16
+; GFX89: v_cvt_f32_f16
+; GFX89: v_fma_f32
+; GFX89: v_add_f32
+define float @fadd_fma_fpext_fmul_f16_to_f32_commute(float %x, float %y, half %u, half %v, float %z) #0 {
+entry:
+  %mul = fmul contract half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fma.f32(float %x, float %y, float %mul.ext)
+  %add = fadd float %z, %fma
+  ret float %add
+}
+
+; fold (fadd x, (fpext (fma y, z, (fmul u, v)))
+;   -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
+
+; GCN-LABEL: {{^}}fadd_fpext_fmuladd_f16_to_f32:
+; GFX9: v_mul_f16
+; GFX9: v_fma_legacy_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_add_f32_e32
+define float @fadd_fpext_fmuladd_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul contract half %u, %v
+  %fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
+  %ext.fma = fpext half %fma to float
+  %add = fadd float %x, %ext.fma
+  ret float %add
+}
+
+; GCN-LABEL: {{^}}fadd_fpext_fma_f16_to_f32:
+; GFX9: v_mul_f16
+; GFX9: v_fma_legacy_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_add_f32_e32
+define float @fadd_fpext_fma_f16_to_f32(float %x, half %y, half %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul contract half %u, %v
+  %fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
+  %ext.fma = fpext half %fma to float
+  %add = fadd float %x, %ext.fma
+  ret float %add
+}
+
+; GCN-LABEL: {{^}}fadd_fpext_fma_f16_to_f32_commute:
+; GFX9: v_mul_f16
+; GFX9: v_fma_legacy_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_add_f32_e32
+define float @fadd_fpext_fma_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul contract half %u, %v
+  %fma = call half @llvm.fma.f16(half %y, half %z, half %mul)
+  %ext.fma = fpext half %fma to float
+  %add = fadd float %ext.fma, %x
+  ret float %add
+}
+
+; fold (fsub (fpext (fmul x, y)), z)
+;   -> (fma (fpext x), (fpext y), (fneg z))
+
+; GCN-LABEL: {{^}}fsub_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %mul.ext = fpext half %mul to float
+  %add = fsub float %mul.ext, %z
+  ret float %add
+}
+
+; fold (fsub x, (fpext (fmul y, z)))
+;   -> (fma (fneg (fpext y)), (fpext z), x)
+
+; GCN-LABEL: {{^}}fsub_fpext_fmul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, -v1, v2, v0 op_sel_hi:[1,1,0]
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_fpext_fmul_f16_to_f32_commute(float %x, half %y, half %z) #0 {
+entry:
+  %mul = fmul contract half %y, %z
+  %mul.ext = fpext half %mul to float
+  %add = fsub contract float %x, %mul.ext
+  ret float %add
+}
+
+; fold (fsub (fpext (fneg (fmul, x, y))), z)
+;   -> (fneg (fma (fpext x), (fpext y), z))
+
+; GCN-LABEL: {{^}}fsub_fpext_fneg_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_fpext_fneg_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %neg.mul = fsub half -0.0, %mul
+  %neg.mul.ext = fpext half %neg.mul to float
+  %add = fsub float %neg.mul.ext, %z
+  ret float %add
+}
+
+; fold (fsub (fneg (fpext (fmul, x, y))), z)
+;   -> (fneg (fma (fpext x)), (fpext y), z)
+
+; GCN-LABEL: {{^}}fsub_fneg_fpext_fmul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, v0, -v1, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e64 v0, v0, -v1
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_fneg_fpext_fmul_f16_to_f32(half %x, half %y, float %z) #0 {
+entry:
+  %mul = fmul half %x, %y
+  %mul.ext = fpext half %mul to float
+  %neg.mul.ext = fsub float -0.0, %mul.ext
+  %add = fsub float %neg.mul.ext, %z
+  ret float %add
+}
+
+; fold (fsub (fmad x, y, (fpext (fmul u, v))), z)
+;    -> (fmad x, y (fmad (fpext u), (fpext v), (fneg z)))
+; GCN-LABEL: {{^}}fsub_muladd_fpext_mul_f16_to_f32:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v2, v3, v4, -v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1
+; GFX9-F32FLUSH-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX9-F32DENORM-NEXT: v_fma_f32 v0, v0, v1, v3
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_muladd_fpext_mul_f16_to_f32(float %x, float %y, float %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fmuladd.f32(float %x, float %y, float %mul.ext)
+  %add = fsub float %fma, %z
+  ret float %add
+}
+
+;  fold (fsub (fpext (fmad x, y, (fmul u, v))), z)
+;    -> (fmad (fpext x), (fpext y),
+;            (fmad (fpext u), (fpext v), (fneg z)))
+
+; GCN-LABEL: {{^}}fsub_fpext_muladd_mul_f16_to_f32:
+; GFX9: v_mul_f16
+; GFX9: v_fma_legacy_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_sub_f32
+; GCN: s_setpc_b64
+define float @fsub_fpext_muladd_mul_f16_to_f32(half %x, half %y, float %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %fma = call half @llvm.fmuladd.f16(half %x, half %y, half %mul)
+  %fma.ext = fpext half %fma to float
+  %add = fsub float %fma.ext, %z
+  ret float %add
+}
+
+; fold (fsub x, (fmad y, z, (fpext (fmul u, v))))
+;   -> (fmad (fneg y), z, (fmad (fneg (fpext u)), (fpext v), x))
+; GCN-LABEL: {{^}}fsub_muladd_fpext_mul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX9-F32FLUSH-NEXT: v_mad_mix_f32 v0, -v3, v4, v0 op_sel_hi:[1,1,0]{{$}}
+; GFX9-F32FLUSH-NEXT: v_mad_f32 v0, -v1, v2, v0{{$}}
+; GFX9-F32FLUSH-NEXT: s_setpc_b64
+
+; GFX9-F32DENORM-NEXT: v_mul_f16_e32 v3, v3, v4
+; GFX9-F32DENORM-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX9-F32DENORM-NEXT: v_fma_f32 v1, v1, v2, v3
+; GFX9-F32DENORM-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-F32DENORM-NEXT: s_setpc_b64
+define float @fsub_muladd_fpext_mul_f16_to_f32_commute(float %x, float %y, float %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %mul.ext = fpext half %mul to float
+  %fma = call float @llvm.fmuladd.f32(float %y, float %z, float %mul.ext)
+  %add = fsub float %x, %fma
+  ret float %add
+}
+
+; fold (fsub x, (fpext (fma y, z, (fmul u, v))))
+;    -> (fma (fneg (fpext y)), (fpext z),
+;            (fma (fneg (fpext u)), (fpext v), x))
+; GCN-LABEL: {{^}}fsub_fpext_muladd_mul_f16_to_f32_commute:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mul_f16_e32 v3, v3, v4
+; GFX9-NEXT: v_fma_legacy_f16 v1, v1, v2, v3
+; GFX9-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX9-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX9-NEXT: s_setpc_b64
+define float @fsub_fpext_muladd_mul_f16_to_f32_commute(float %x, half %y, half %z, half %u, half %v) #0 {
+entry:
+  %mul = fmul half %u, %v
+  %fma = call half @llvm.fmuladd.f16(half %y, half %z, half %mul)
+  %fma.ext = fpext half %fma to float
+  %add = fsub float %x, %fma.ext
+  ret float %add
+}
+
+declare float @llvm.fmuladd.f32(float, float, float) #0
+declare float @llvm.fma.f32(float, float, float) #0
+declare half @llvm.fmuladd.f16(half, half, half) #0
+declare half @llvm.fma.f16(half, half, half) #0
+
+attributes #0 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/fpext.f16.ll b/test/CodeGen/AMDGPU/fpext.f16.ll
index ec19fd199b4ec..84ff7e0209226 100644
--- a/test/CodeGen/AMDGPU/fpext.f16.ll
+++ b/test/CodeGen/AMDGPU/fpext.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
 
 ; GCN-LABEL: {{^}}fpext_f16_to_f32
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
@@ -180,7 +180,7 @@ entry:
 ; GCN-DAG: v_and_b32_e32 [[XOR:v[0-9]+]], 0x7fff, [[A]]
 
 ; SI-DAG: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[A]]
-; VI-DAG: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], |[[A]]|
+; GFX89-DAG: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], |[[A]]|
 
 ; GCN: store_dword [[CVT]]
 ; GCN: store_short [[XOR]]
@@ -226,7 +226,7 @@ entry:
 ; GCN-DAG: v_or_b32_e32 [[OR:v[0-9]+]], 0x8000, [[A]]
 
 ; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[OR]]
-; VI-DAG: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -|[[OR]]|
+; GFX89-DAG: v_cvt_f32_f16_e64 [[CVT:v[0-9]+]], -|[[OR]]|
 
 ; GCN: buffer_store_dword [[CVT]]
 ; GCN: buffer_store_short [[OR]]
diff --git a/test/CodeGen/AMDGPU/fptosi.f16.ll b/test/CodeGen/AMDGPU/fptosi.f16.ll
index f593030764a99..57d8c62ec7ce6 100644
--- a/test/CodeGen/AMDGPU/fptosi.f16.ll
+++ b/test/CodeGen/AMDGPU/fptosi.f16.ll
@@ -57,8 +57,8 @@ entry:
 ; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
 ; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
 ; SI: v_cvt_i32_f32_e32 v[[R_I16_0:[0-9]+]], v[[A_F32_0]]
-; SI: v_cvt_i32_f32_e32 v[[R_I16_1:[0-9]+]], v[[A_F32_1]]
-; SI: v_and_b32_e32 v[[R_I16_LO:[0-9]+]], 0xffff, v[[R_I16_0]]
+; SI-DAG: v_cvt_i32_f32_e32 v[[R_I16_1:[0-9]+]], v[[A_F32_1]]
+; SI-DAG: v_and_b32_e32 v[[R_I16_LO:[0-9]+]], 0xffff, v[[R_I16_0]]
 ; SI: v_lshlrev_b32_e32 v[[R_I16_HI:[0-9]+]], 16, v[[R_I16_1]]
 ; SI: v_or_b32_e32 v[[R_V2_I16:[0-9]+]], v[[R_I16_LO]], v[[R_I16_HI]]
 
diff --git a/test/CodeGen/AMDGPU/fptoui.f16.ll b/test/CodeGen/AMDGPU/fptoui.f16.ll
index cebe3304d542b..e2c8aacfb2ac7 100644
--- a/test/CodeGen/AMDGPU/fptoui.f16.ll
+++ b/test/CodeGen/AMDGPU/fptoui.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}fptoui_f16_to_i16
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/fptrunc.f16.ll b/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 64df625d4bb5a..3aff885b7b5ff 100644
--- a/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}fptrunc_f32_to_f16:
 ; GCN: buffer_load_dword v[[A_F32:[0-9]+]]
@@ -65,6 +65,9 @@ entry:
 ; GCN-DAG: v_cvt_f32_f64_e32 v[[A_F32_0:[0-9]+]], v{{\[}}[[A_F64_0]]:{{[0-9]+}}{{\]}}
 ; GCN-DAG: v_cvt_f32_f64_e32 v[[A_F32_1:[0-9]+]], v{{\[}}{{[0-9]+}}:[[A_F64_3]]{{\]}}
 ; GCN-DAG: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[A_F32_0]]
+;
+; SI-DAG: v_cvt_f16_f32_e32 v[[CVTHI:[0-9]+]], v[[A_F32_1]]
+; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[CVTHI]]
 
 ; VI: v_cvt_f16_f32_sdwa v[[R_F16_HI:[0-9]+]], v[[A_F32_1]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
 
diff --git a/test/CodeGen/AMDGPU/frame-index-amdgiz.ll b/test/CodeGen/AMDGPU/frame-index-amdgiz.ll
index dd46403b68af1..e2b1950f8855d 100644
--- a/test/CodeGen/AMDGPU/frame-index-amdgiz.ll
+++ b/test/CodeGen/AMDGPU/frame-index-amdgiz.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn---amdgiz -mcpu=kaveri -verify-machineinstrs < %s | FileCheck %s
 ;
 ; The original OpenCL kernel:
 ; kernel void f(global int *a, int i,  int j) {
@@ -6,15 +6,14 @@
 ;  x[i] = 7;
 ;  a[0] = x[j];
 ; }
-; clang -cc1 -triple amdgcn---amdgizcl -emit-llvm -o - 
+; clang -cc1 -triple amdgcn---amdgizcl -emit-llvm -o -
 
 target datalayout = "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5"
-target triple = "amdgcn---amdgiz"
 
 define amdgpu_kernel void @f(i32 addrspace(1)* nocapture %a, i32 %i, i32 %j) local_unnamed_addr #0 {
 entry:
-; CHECK: s_load_dword s2, s[0:1], 0xb
 ; CHECK: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CHECK: s_load_dword s2, s[0:1], 0xb
 ; CHECK: s_load_dword s0, s[0:1], 0xc
 ; CHECK: s_mov_b32 s8, SCRATCH_RSRC_DWORD0
 ; CHECK: s_mov_b32 s9, SCRATCH_RSRC_DWORD1
@@ -28,9 +27,9 @@ entry:
 ; CHECK: s_lshl_b32 s0, s0, 2
 ; CHECK: buffer_store_dword v2, v1, s[8:11], s3 offen
 ; CHECK: v_add_i32_e32 v0, vcc, s0, v0
-; CHECK: buffer_load_dword v0, v0, s[8:11], s3 offen
 ; CHECK: s_mov_b32 s7, 0xf000
 ; CHECK: s_mov_b32 s6, -1
+; CHECK: buffer_load_dword v0, v0, s[8:11], s3 offen
 ; CHECK: s_waitcnt vmcnt(0)
 ; CHECK: buffer_store_dword v0, off, s[4:7], 0
 ; CHECK: s_endpgm
diff --git a/test/CodeGen/AMDGPU/frame-index-elimination.ll b/test/CodeGen/AMDGPU/frame-index-elimination.ll
index 25cbb7b105f04..9b75c44eac050 100644
--- a/test/CodeGen/AMDGPU/frame-index-elimination.ll
+++ b/test/CodeGen/AMDGPU/frame-index-elimination.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; Test that non-entry function frame indices are expanded properly to
 ; give an index relative to the scratch wave offset register
@@ -7,8 +8,13 @@
 ; GCN-LABEL: {{^}}func_mov_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_mov_fi_i32() #0 {
@@ -23,9 +29,16 @@ define void @func_mov_fi_i32() #0 {
 ; GCN-LABEL: {{^}}func_add_constant_to_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+; CI-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+; GFX9-NEXT: v_add_u32_e32 v0, 4, v0
+
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_add_constant_to_fi_i32() #0 {
@@ -40,8 +53,13 @@ define void @func_add_constant_to_fi_i32() #0 {
 
 ; GCN-LABEL: {{^}}func_other_fi_user_i32:
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+
 ; GCN-NEXT: v_mul_lo_i32 v0, v0, 9
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
@@ -72,9 +90,16 @@ define void @func_load_private_arg_i32_ptr(i32* %ptr) #0 {
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_mov_b32 s5, s32
-; GCN-NEXT: s_sub_u32 [[SUB:s[0-9]+]], s5, s4
-; GCN-NEXT: v_lshr_b32_e64 v0, [[SUB]], 6
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GCN-NEXT: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s5, s4
+
+; CI-NEXT: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
+; CI-NEXT: v_add_i32_e64 [[ADD:v[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
+; CI-NEXT: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, [[SUB_OFFSET]]
+; GFX9-NEXT: v_add_u32_e32 [[ADD:v[0-9]+]], 4, [[SHIFT]]
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[ADD]]
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @void_func_byval_struct_i8_i32_ptr({ i8, i32 }* byval %arg0) #0 {
@@ -100,13 +125,26 @@ define void @void_func_byval_struct_i8_i32_ptr_value({ i8, i32 }* byval %arg0) #
   ret void
 }
 
+; FIXME: Should be able to see that this can use vaddr, but the
+; FrameIndex is hidden behind a CopyFromReg in the second block.
+
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr_nonentry_block:
-; GCN: s_sub_u32 s6, s5, s4
-; GCN: v_lshr_b32_e64 v1, s6, 6
+; GCN: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s5, s4
+
+; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
+; CI: v_add_i32_e64 [[ADD:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
+
+; GFX9: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, [[SUB_OFFSET]]
+; GFX9: v_add_u32_e32 [[ADD:v[0-9]+]], 4, [[SHIFT]]
+
 ; GCN: s_and_saveexec_b64
 
-; GCN: v_add_i32_e32 v0, vcc, 4, v1
-; GCN: buffer_load_dword v1, v1, s[0:3], s4 offen offset:4
+; CI: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+; CI: buffer_load_dword v1, v0, s[0:3], s4 offen{{$}}
+
+; GFX9: v_add_u32_e32 v0, 4, [[ADD]]
+; GFX9: buffer_load_dword v1, v{{[0-9]+}}, s[0:3], s4 offen offset:4{{$}}
+
 ; GCN: ds_write_b32
 define void @void_func_byval_struct_i8_i32_ptr_nonentry_block({ i8, i32 }* byval %arg0, i32 %arg2) #0 {
   %cmp = icmp eq i32 %arg2, 0
@@ -126,9 +164,14 @@ ret:
 ; Added offset can't be used with VOP3 add
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32:
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
 ; GCN-DAG: s_movk_i32 s6, 0x204
-; GCN: v_add_i32_e64 v0, s[6:7], s6, [[SCALED]]
+
+; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI: v_add_i32_e64 v0, s[6:7], s6, [[SCALED]]
+
+; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9: v_add_u32_e32 v0, s6, [[SCALED]]
+
 ; GCN: v_mul_lo_i32 v0, v0, 9
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_non_inline_imm_offset_i32() #0 {
@@ -142,11 +185,17 @@ define void @func_other_fi_user_non_inline_imm_offset_i32() #0 {
   store volatile i32 %mul, i32 addrspace(3)* undef
   ret void
 }
+
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32_vcc_live:
 ; GCN: s_sub_u32 [[DIFF:s[0-9]+]], s5, s4
-; GCN-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
 ; GCN-DAG: s_movk_i32 [[OFFSET:s[0-9]+]], 0x204
-; GCN: v_add_i32_e64 v0, s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
+
+; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
+; CI: v_add_i32_e64 v0, s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
+
+; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[DIFF]]
+; GFX9: v_add_u32_e32 v0, [[OFFSET]], [[SCALED]]
+
 ; GCN: v_mul_lo_i32 v0, v0, 9
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_non_inline_imm_offset_i32_vcc_live() #0 {
@@ -163,4 +212,52 @@ define void @func_other_fi_user_non_inline_imm_offset_i32_vcc_live() #0 {
   ret void
 }
 
+declare void @func(<4 x float>* nocapture) #0
+
+; undef flag not preserved in eliminateFrameIndex when handling the
+; stores in the middle block.
+
+; GCN-LABEL: {{^}}undefined_stack_store_reg:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_store_dword v0, off, s[0:3], s5 offset:
+; GCN: buffer_store_dword v0, off, s[0:3], s5 offset:
+; GCN: buffer_store_dword v0, off, s[0:3], s5 offset:
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s5 offset:
+define void @undefined_stack_store_reg(float %arg, i32 %arg1) #0 {
+bb:
+  %tmp = alloca <4 x float>, align 16
+  %tmp2 = insertelement <4 x float> undef, float %arg, i32 0
+  store <4 x float> %tmp2, <4 x float>* undef
+  %tmp3 = icmp eq i32 %arg1, 0
+  br i1 %tmp3, label %bb4, label %bb5
+
+bb4:
+  call void @func(<4 x float>* nonnull undef)
+  store <4 x float> %tmp2, <4 x float>* %tmp, align 16
+  call void @func(<4 x float>* nonnull %tmp)
+  br label %bb5
+
+bb5:
+  ret void
+}
+
+; GCN-LABEL: {{^}}alloca_ptr_nonentry_block:
+; GCN: s_and_saveexec_b64
+; GCN: buffer_load_dword v{{[0-9]+}}, off, s[0:3], s5 offset:12
+define void @alloca_ptr_nonentry_block(i32 %arg0) #0 {
+  %alloca0 = alloca { i8, i32 }, align 4
+  %cmp = icmp eq i32 %arg0, 0
+  br i1 %cmp, label %bb, label %ret
+
+bb:
+  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %alloca0, i32 0, i32 0
+  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %alloca0, i32 0, i32 1
+  %load1 = load volatile i32, i32* %gep1
+  store volatile i32* %gep1, i32* addrspace(3)* undef
+  br label %ret
+
+ret:
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/fsub.f16.ll b/test/CodeGen/AMDGPU/fsub.f16.ll
index 15a4ce2d88f7d..fc055a58e7575 100644
--- a/test/CodeGen/AMDGPU/fsub.f16.ll
+++ b/test/CodeGen/AMDGPU/fsub.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=VI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=VI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}fsub_f16:
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/ftrunc.f64.ll b/test/CodeGen/AMDGPU/ftrunc.f64.ll
index bb2a6ba8e3483..6fc4c8b7d243b 100644
--- a/test/CodeGen/AMDGPU/ftrunc.f64.ll
+++ b/test/CodeGen/AMDGPU/ftrunc.f64.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
 
 declare double @llvm.trunc.f64(double) nounwind readnone
 declare <2 x double> @llvm.trunc.v2f64(<2 x double>) nounwind readnone
diff --git a/test/CodeGen/AMDGPU/function-args.ll b/test/CodeGen/AMDGPU/function-args.ll
index 6b22cb0b7e286..ca36732540b25 100644
--- a/test/CodeGen/AMDGPU/function-args.ll
+++ b/test/CodeGen/AMDGPU/function-args.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89 %s
 
 ; GCN-LABEL: {{^}}void_func_i1:
 ; GCN: v_and_b32_e32 v0, 1, v0
@@ -24,7 +24,7 @@ define void @void_func_i1_zeroext(i1 zeroext %arg0) #0 {
 
 ; GCN-LABEL: {{^}}void_func_i1_signext:
 ; GCN: s_waitcnt
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 12, v0
+; GCN-NEXT: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}12, v0
 ; GCN-NOT: v0
 ; GCN: buffer_store_dword v0, off
 define void @void_func_i1_signext(i1 signext %arg0) #0 {
@@ -60,7 +60,7 @@ define void @void_func_i8(i8 %arg0) #0 {
 
 ; GCN-LABEL: {{^}}void_func_i8_zeroext:
 ; GCN-NOT: and_b32
-; GCN: v_add_i32_e32 v0, vcc, 12, v0
+; GCN: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}12, v0
 define void @void_func_i8_zeroext(i8 zeroext %arg0) #0 {
   %ext = zext i8 %arg0 to i32
   %add = add i32 %ext, 12
@@ -70,7 +70,7 @@ define void @void_func_i8_zeroext(i8 zeroext %arg0) #0 {
 
 ; GCN-LABEL: {{^}}void_func_i8_signext:
 ; GCN-NOT: v_bfe_i32
-; GCN: v_add_i32_e32 v0, vcc, 12, v0
+; GCN: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}12, v0
 define void @void_func_i8_signext(i8 signext %arg0) #0 {
   %ext = sext i8 %arg0 to i32
   %add = add i32 %ext, 12
@@ -87,7 +87,7 @@ define void @void_func_i16(i16 %arg0) #0 {
 
 ; GCN-LABEL: {{^}}void_func_i16_zeroext:
 ; GCN-NOT: v0
-; GCN: v_add_i32_e32 v0, vcc, 12, v0
+; GCN: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}12, v0
 define void @void_func_i16_zeroext(i16 zeroext %arg0) #0 {
   %ext = zext i16 %arg0 to i32
   %add = add i32 %ext, 12
@@ -97,7 +97,7 @@ define void @void_func_i16_zeroext(i16 zeroext %arg0) #0 {
 
 ; GCN-LABEL: {{^}}void_func_i16_signext:
 ; GCN-NOT: v0
-; GCN: v_add_i32_e32 v0, vcc, 12, v0
+; GCN: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}12, v0
 define void @void_func_i16_signext(i16 signext %arg0) #0 {
   %ext = sext i16 %arg0 to i32
   %add = add i32 %ext, 12
@@ -502,8 +502,8 @@ define void @void_func_struct_i8_i32({ i8, i32 } %arg0) #0 {
 }
 
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32:
-; GCN-DAG: buffer_load_ubyte v[[ELT0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[ELT1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_ubyte v[[ELT0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[ELT1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
 ; GCN-DAG: buffer_store_dword v[[ELT1]]
 ; GCN-DAG: buffer_store_byte v[[ELT0]]
 define void @void_func_byval_struct_i8_i32({ i8, i32 }* byval %arg0) #0 {
@@ -513,10 +513,10 @@ define void @void_func_byval_struct_i8_i32({ i8, i32 }* byval %arg0) #0 {
 }
 
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_x2:
-; GCN: buffer_load_ubyte v[[ELT0_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN: buffer_load_dword v[[ELT1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN: buffer_load_ubyte v[[ELT0_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN: buffer_load_dword v[[ELT1_1:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN: buffer_load_ubyte v[[ELT0_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN: buffer_load_dword v[[ELT1_0:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN: buffer_load_ubyte v[[ELT0_1:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN: buffer_load_dword v[[ELT1_1:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
 ; GCN: ds_write_b32 v0, v0
 ; GCN: s_setpc_b64
@@ -530,7 +530,7 @@ define void @void_func_byval_struct_i8_i32_x2({ i8, i32 }* byval %arg0, { i8, i3
 }
 
 ; GCN-LABEL: {{^}}void_func_byval_i32_byval_i64:
-; GCN-DAG: buffer_load_dword v[[ARG0_LOAD:[0-9]+]], off, s[0:3], s5{{$}}
+; GCN-DAG: buffer_load_dword v[[ARG0_LOAD:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
 ; GCN-DAG: buffer_load_dword v[[ARG1_LOAD0:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
 ; GCN-DAG: buffer_load_dword v[[ARG1_LOAD1:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
 ; GCN-DAG: buffer_store_dword v[[ARG0_LOAD]], off
@@ -552,9 +552,9 @@ define void @void_func_byval_i32_byval_i64(i32* byval %arg0, i64* byval %arg1) #
 ; GCN-DAG: buffer_store_dwordx4 v[20:23], off
 ; GCN-DAG: buffer_store_dwordx4 v[24:27], off
 ; GCN-DAG: buffer_store_dwordx4 v[28:31], off
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:4
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:8
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:8
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:12
 
 ; GCN: buffer_store_dword v[[LOAD_ARG1]]
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LOAD_ARG2_0]]:[[LOAD_ARG2_1]]{{\]}}, off
@@ -567,14 +567,14 @@ define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0
 
 ; FIXME: Different ext load types on CI vs. VI
 ; GCN-LABEL: {{^}}void_func_v32i32_i1_i8_i16:
-; GCN-DAG: buffer_load_ubyte [[LOAD_ARG1:v[0-9]+]], off, s[0:3], s5{{$}}
-; VI-DAG: buffer_load_ushort [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; VI-DAG: buffer_load_ushort [[LOAD_ARG3:v[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; VI-DAG: buffer_load_ushort [[LOAD_ARG4:v[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_ubyte [[LOAD_ARG1:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; VI-DAG: buffer_load_ushort [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; VI-DAG: buffer_load_ushort [[LOAD_ARG3:v[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; VI-DAG: buffer_load_ushort [[LOAD_ARG4:v[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
-; CI-DAG: buffer_load_dword [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; CI-DAG: buffer_load_dword [[LOAD_ARG3:v[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; CI-DAG: buffer_load_dword [[LOAD_ARG4:v[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; CI-DAG: buffer_load_dword [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; CI-DAG: buffer_load_dword [[LOAD_ARG3:v[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; CI-DAG: buffer_load_dword [[LOAD_ARG4:v[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
 ; GCN-DAG: v_and_b32_e32 [[TRUNC_ARG1_I1:v[0-9]+]], 1, [[LOAD_ARG1]]
 ; CI-DAG: v_cvt_f16_f32_e32 [[CVT_ARG4:v[0-9]+]], [[LOAD_ARG4]]
@@ -582,7 +582,7 @@ define void @void_func_v32i32_i32_i64(<32 x i32> %arg0, i32 %arg1, i64 %arg2) #0
 ; GCN: buffer_store_byte [[TRUNC_ARG1_I1]], off
 ; GCN: buffer_store_byte [[LOAD_ARG2]], off
 ; GCN: buffer_store_short [[LOAD_ARG3]], off
-; VI: buffer_store_short [[LOAD_ARG4]], off
+; GFX89 buffer_store_short [[LOAD_ARG4]], off
 
 ; CI: buffer_store_short [[CVT_ARG4]], off
 define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i16 %arg3, half %arg4) #0 {
@@ -595,10 +595,10 @@ define void @void_func_v32i32_i1_i8_i16(<32 x i32> %arg0, i1 %arg1, i8 %arg2, i1
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v2i32_v2f32:
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LOAD_ARG1_0]]:[[LOAD_ARG1_1]]{{\]}}, off
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LOAD_ARG2_0]]:[[LOAD_ARG2_1]]{{\]}}, off
@@ -610,8 +610,8 @@ define void @void_func_v32i32_v2i32_v2f32(<32 x i32> %arg0, <2 x i32> %arg1, <2
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v2i16_v2f16:
-; GFX9-DAG: buffer_load_dword [[LOAD_ARG1:v[0-9]+]], off, s[0:3], s5{{$}}
-; GFX9-DAG: buffer_load_dword [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GFX9-DAG: buffer_load_dword [[LOAD_ARG1:v[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GFX9-DAG: buffer_load_dword [[LOAD_ARG2:v[0-9]+]], off, s[0:3], s5 offset:8{{$}}
 ; GFX9: buffer_store_dword [[LOAD_ARG1]], off
 ; GFX9: buffer_store_short [[LOAD_ARG2]], off
 define void @void_func_v32i32_v2i16_v2f16(<32 x i32> %arg0, <2 x i16> %arg1, <2 x half> %arg2) #0 {
@@ -622,15 +622,15 @@ define void @void_func_v32i32_v2i16_v2f16(<32 x i32> %arg0, <2 x i16> %arg1, <2
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v2i64_v2f64:
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
 
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG1_0]]:[[LOAD_ARG1_3]]{{\]}}, off
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG2_0]]:[[LOAD_ARG2_3]]{{\]}}, off
@@ -642,15 +642,15 @@ define void @void_func_v32i32_v2i64_v2f64(<32 x i32> %arg0, <2 x i64> %arg1, <2
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v4i32_v4f32:
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
 
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
 
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG1_0]]:[[LOAD_ARG1_3]]{{\]}}, off
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG2_0]]:[[LOAD_ARG2_3]]{{\]}}, off
@@ -662,23 +662,23 @@ define void @void_func_v32i32_v4i32_v4f32(<32 x i32> %arg0, <4 x i32> %arg1, <4
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v8i32_v8f32:
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_4:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_5:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_6:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_7:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_4:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_5:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_6:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_7:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
 
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:36{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:40{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:44{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_4:[0-9]+]], off, s[0:3], s5 offset:48{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_5:[0-9]+]], off, s[0:3], s5 offset:52{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_6:[0-9]+]], off, s[0:3], s5 offset:56{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_7:[0-9]+]], off, s[0:3], s5 offset:60{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:36{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:40{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:44{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:48{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_4:[0-9]+]], off, s[0:3], s5 offset:52{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_5:[0-9]+]], off, s[0:3], s5 offset:56{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_6:[0-9]+]], off, s[0:3], s5 offset:60{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_7:[0-9]+]], off, s[0:3], s5 offset:64{{$}}
 
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG1_4]]:[[LOAD_ARG1_7]]{{\]}}, off
 ; GCN: buffer_store_dwordx4 v{{\[}}[[LOAD_ARG1_0]]:[[LOAD_ARG1_3]]{{\]}}, off
@@ -692,39 +692,39 @@ define void @void_func_v32i32_v8i32_v8f32(<32 x i32> %arg0, <8 x i32> %arg1, <8
 }
 
 ; GCN-LABEL: {{^}}void_func_v32i32_v16i32_v16f32:
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_3:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_4:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_5:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_6:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_7:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_8:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_9:[0-9]+]], off, s[0:3], s5 offset:36{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_10:[0-9]+]], off, s[0:3], s5 offset:40{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_11:[0-9]+]], off, s[0:3], s5 offset:44{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_12:[0-9]+]], off, s[0:3], s5 offset:48{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_13:[0-9]+]], off, s[0:3], s5 offset:52{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_14:[0-9]+]], off, s[0:3], s5 offset:56{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_15:[0-9]+]], off, s[0:3], s5 offset:60{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_0:[0-9]+]], off, s[0:3], s5 offset:4{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_1:[0-9]+]], off, s[0:3], s5 offset:8{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_2:[0-9]+]], off, s[0:3], s5 offset:12{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_:[0-9]+]], off, s[0:3], s5 offset:16{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_4:[0-9]+]], off, s[0:3], s5 offset:20{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_5:[0-9]+]], off, s[0:3], s5 offset:24{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_6:[0-9]+]], off, s[0:3], s5 offset:28{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_7:[0-9]+]], off, s[0:3], s5 offset:32{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_8:[0-9]+]], off, s[0:3], s5 offset:36{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_9:[0-9]+]], off, s[0:3], s5 offset:40{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_10:[0-9]+]], off, s[0:3], s5 offset:44{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_11:[0-9]+]], off, s[0:3], s5 offset:48{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_12:[0-9]+]], off, s[0:3], s5 offset:52{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_13:[0-9]+]], off, s[0:3], s5 offset:56{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG1_14:[0-9]+]], off, s[0:3], s5 offset:60{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_15:[0-9]+]], off, s[0:3], s5 offset:64{{$}}
 
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:64{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:68{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:72{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:76{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_4:[0-9]+]], off, s[0:3], s5 offset:80{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_5:[0-9]+]], off, s[0:3], s5 offset:84{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_6:[0-9]+]], off, s[0:3], s5 offset:88{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_7:[0-9]+]], off, s[0:3], s5 offset:92{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_8:[0-9]+]], off, s[0:3], s5 offset:96{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_9:[0-9]+]], off, s[0:3], s5 offset:100{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_10:[0-9]+]], off, s[0:3], s5 offset:104{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_11:[0-9]+]], off, s[0:3], s5 offset:108{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_12:[0-9]+]], off, s[0:3], s5 offset:112{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_13:[0-9]+]], off, s[0:3], s5 offset:116{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_14:[0-9]+]], off, s[0:3], s5 offset:120{{$}}
-; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_15:[0-9]+]], off, s[0:3], s5 offset:124{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_0:[0-9]+]], off, s[0:3], s5 offset:68{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_1:[0-9]+]], off, s[0:3], s5 offset:72{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_2:[0-9]+]], off, s[0:3], s5 offset:76{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_3:[0-9]+]], off, s[0:3], s5 offset:80{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_4:[0-9]+]], off, s[0:3], s5 offset:84{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_5:[0-9]+]], off, s[0:3], s5 offset:88{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_6:[0-9]+]], off, s[0:3], s5 offset:92{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_7:[0-9]+]], off, s[0:3], s5 offset:96{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_8:[0-9]+]], off, s[0:3], s5 offset:100{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_9:[0-9]+]], off, s[0:3], s5 offset:104{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_10:[0-9]+]], off, s[0:3], s5 offset:108{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_11:[0-9]+]], off, s[0:3], s5 offset:112{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_12:[0-9]+]], off, s[0:3], s5 offset:116{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_13:[0-9]+]], off, s[0:3], s5 offset:120{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_14:[0-9]+]], off, s[0:3], s5 offset:124{{$}}
+; GCN-DAG: buffer_load_dword v[[LOAD_ARG2_15:[0-9]+]], off, s[0:3], s5 offset:128{{$}}
 define void @void_func_v32i32_v16i32_v16f32(<32 x i32> %arg0, <16 x i32> %arg1, <16 x float> %arg2) #0 {
   store volatile <32 x i32> %arg0, <32 x i32> addrspace(1)* undef
   store volatile <16 x i32> %arg1, <16 x i32> addrspace(1)* undef
diff --git a/test/CodeGen/AMDGPU/function-returns.ll b/test/CodeGen/AMDGPU/function-returns.ll
index f704d43a1742c..4a24f5e285b9c 100644
--- a/test/CodeGen/AMDGPU/function-returns.ll
+++ b/test/CodeGen/AMDGPU/function-returns.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=hawaii -verify-machineinstrs < %s | FileCheck  -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX89,GFX9 %s
 
 ; GCN-LABEL: {{^}}i1_func_void:
 ; GCN: buffer_load_ubyte v0, off
@@ -283,8 +283,9 @@ define <2 x i16> @v2i16_func_void() #0 {
 
 ; GCN-LABEL: {{^}}v3i16_func_void:
 ; GFX9: buffer_load_dwordx2 v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64
+; GFX9: s_waitcnt vmcnt(0)
+; GFX9: v_lshrrev_b32
+; GFX9: s_setpc_b64
 define <3 x i16> @v3i16_func_void() #0 {
   %val = load <3 x i16>, <3 x i16> addrspace(1)* undef
   ret <3 x i16> %val
@@ -305,7 +306,7 @@ define <4 x i16> @v4i16_func_void() #0 {
 ; GFX9: buffer_load_ushort v4
 ; GFX9: v_lshrrev_b32_e32 v3, 16, v1
 ; GFX9: v_mov_b32_e32 v2, v1
-; GFX9: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9: v_lshrrev_b32_e32 v1, 16, v0
 ; GCN: s_setpc_b64
 define <5 x i16> @v5i16_func_void() #0 {
   %ptr = load volatile <5 x i16> addrspace(1)*, <5 x i16> addrspace(1)* addrspace(2)* undef
@@ -352,7 +353,7 @@ define <16 x i8> @v16i8_func_void() #0 {
 ; GCN-DAG: v_lshrrev_b32_e32 v2, 16, v0
 ; GCN-DAG: v_lshrrev_b32_e32 v3, 24, v0
 ; CI-DAG: v_bfe_u32 v1, v0, 8, 8
-; VI-DAG: v_lshrrev_b16_e32 v1, 8, v0
+; GFX89-DAG: v_lshrrev_b16_e32 v1, 8, v0
 ; GCN: s_setpc_b64
 define <4  x i8> @v4i8_func_void() #0 {
   %ptr = load volatile <4  x i8> addrspace(1)*, <4  x i8> addrspace(1)* addrspace(2)* undef
@@ -375,52 +376,56 @@ define {i8, i32} @struct_i8_i32_func_void() #0 {
 ; GCN: buffer_load_dword [[VAL1:v[0-9]+]]
 ; GCN: buffer_store_byte [[VAL0]], v0, s[0:3], s4 offen{{$}}
 ; GCN: buffer_store_dword [[VAL1]], v0, s[0:3], s4 offen offset:4{{$}}
-define void @void_func_sret_struct_i8_i32({ i8, i32 }* sret %arg0) #0 {
+define void @void_func_sret_struct_i8_i32({ i8, i32 } addrspace(5)* sret %arg0) #0 {
   %val0 = load volatile i8, i8 addrspace(1)* undef
   %val1 = load volatile i32, i32 addrspace(1)* undef
-  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 0
-  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 1
-  store i8 %val0, i8* %gep0
-  store i32 %val1, i32* %gep1
+  %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %arg0, i32 0, i32 0
+  %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 } addrspace(5)* %arg0, i32 0, i32 1
+  store i8 %val0, i8 addrspace(5)* %gep0
+  store i32 %val1, i32 addrspace(5)* %gep1
   ret void
 }
 
+; FIXME: Should be able to fold offsets in all of these pre-gfx9. Call
+; lowering introduces an extra CopyToReg/CopyFromReg obscuring the
+; AssertZext inserted. Not using it introduces the spills.
+
 ; GCN-LABEL: {{^}}v33i32_func_void:
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:4{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:8{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:12{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:16{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:20{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:24{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:28{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:32{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:36{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:40{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:44{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:48{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:52{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:56{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:60{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:64{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:68{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:72{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:76{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:80{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:84{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:88{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:92{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:96{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:100{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:104{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:108{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:112{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:116{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:120{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:124{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
-; GCN: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:4{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:8{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:12{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:16{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:20{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:24{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:28{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:32{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:36{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:40{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:44{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:48{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:52{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:56{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:60{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:64{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:68{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:72{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:76{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:80{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:84{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:88{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:92{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:96{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:100{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:104{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:108{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:112{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:116{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:120{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:124{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
+; GFX9: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64
 define <33 x i32> @v33i32_func_void() #0 {
   %ptr = load volatile <33 x i32> addrspace(1)*, <33 x i32> addrspace(1)* addrspace(2)* undef
   %val = load <33 x i32>, <33 x i32> addrspace(1)* %ptr
@@ -428,41 +433,41 @@ define <33 x i32> @v33i32_func_void() #0 {
 }
 
 ; GCN-LABEL: {{^}}struct_v32i32_i32_func_void:
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:4{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:8{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:12{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:16{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:20{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:24{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:28{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:32{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:36{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:40{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:44{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:48{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:52{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:56{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:60{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:64{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:68{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:72{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:76{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:80{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:84{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:88{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:92{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:96{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:100{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:104{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:108{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:112{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:116{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:120{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:124{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
-; GCN: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:4{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:8{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:12{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:16{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:20{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:24{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:28{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:32{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:36{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:40{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:44{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:48{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:52{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:56{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:60{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:64{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:68{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:72{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:76{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:80{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:84{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:88{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:92{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:96{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:100{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:104{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:108{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:112{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:116{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:120{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:124{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
+; GFX9: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64
 define { <32 x i32>, i32 } @struct_v32i32_i32_func_void() #0 {
   %ptr = load volatile { <32 x i32>, i32 } addrspace(1)*, { <32 x i32>, i32 } addrspace(1)* addrspace(2)* undef
   %val = load { <32 x i32>, i32 }, { <32 x i32>, i32 } addrspace(1)* %ptr
@@ -470,41 +475,41 @@ define { <32 x i32>, i32 } @struct_v32i32_i32_func_void() #0 {
 }
 
 ; GCN-LABEL: {{^}}struct_i32_v32i32_func_void:
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:132{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:136{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:140{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:144{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:148{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:152{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:156{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:160{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:164{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:168{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:172{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:176{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:180{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:184{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:188{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:192{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:196{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:200{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:204{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:208{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:212{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:216{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:220{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:224{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:228{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:232{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:236{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:240{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:244{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:248{{$}}
-; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:252{{$}}
-; GCN: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_setpc_b64
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:128{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:132{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:136{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:140{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:144{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:148{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:152{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:156{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:160{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:164{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:168{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:172{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:176{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:180{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:184{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:188{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:192{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:196{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:200{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:204{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:208{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:212{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:216{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:220{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:224{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:228{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:232{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:236{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:240{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:244{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:248{{$}}
+; GFX9-DAG: buffer_store_dword v{{[0-9]+}}, v0, s[0:3], s4 offen offset:252{{$}}
+; GFX9: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64
 define { i32, <32 x i32> } @struct_i32_v32i32_func_void() #0 {
   %ptr = load volatile { i32, <32 x i32> } addrspace(1)*, { i32, <32 x i32> } addrspace(1)* addrspace(2)* undef
   %val = load { i32, <32 x i32> }, { i32, <32 x i32> } addrspace(1)* %ptr
diff --git a/test/CodeGen/AMDGPU/global_atomics.ll b/test/CodeGen/AMDGPU/global_atomics.ll
index 6928bede547ef..1c22c375bc50d 100644
--- a/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/test/CodeGen/AMDGPU/global_atomics.ll
@@ -1,8 +1,10 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
-; FUNC-LABEL: {{^}}atomic_add_i32_offset:
-; GCN: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_add_i32_offset:
+; SIVI: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_add_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -10,9 +12,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_soffset:
-; GCN: s_mov_b32 [[SREG:s[0-9]+]], 0x8ca0
-; GCN: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
+; GCN-LABEL: {{^}}atomic_add_i32_max_neg_offset:
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:-4096{{$}}
+define amdgpu_kernel void @atomic_add_i32_max_neg_offset(i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i64 -1024
+  %val = atomicrmw volatile add i32 addrspace(1)* %gep, i32 %in seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}atomic_add_i32_soffset:
+; SIVI: s_mov_b32 [[SREG:s[0-9]+]], 0x8ca0
+; SIVI: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
+
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i32_soffset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 9000
@@ -20,11 +33,14 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_huge_offset:
+; GCN-LABEL: {{^}}atomic_add_i32_huge_offset:
 ; SI-DAG: v_mov_b32_e32 v[[PTRLO:[0-9]+]], 0xdeac
 ; SI-DAG: v_mov_b32_e32 v[[PTRHI:[0-9]+]], 0xabcd
 ; SI: buffer_atomic_add v{{[0-9]+}}, v{{\[}}[[PTRLO]]:[[PTRHI]]{{\]}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+
 ; VI: flat_atomic_add
+
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i32_huge_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 47224239175595
@@ -33,9 +49,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_offset:
-; GCN: buffer_atomic_add [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_add_i32_ret_offset:
+; SIVI: buffer_atomic_add [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_add_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -44,9 +62,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_add_i32_addr64_offset:
 ; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_add_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -55,10 +74,13 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_add_i32_ret_addr64_offset:
 ; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -68,17 +90,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32:
-; GCN: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_add_i32:
+; SIVI: buffer_atomic_add v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_add v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_ret:
-; GCN: buffer_atomic_add [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_add_i32_ret:
+; SIVI: buffer_atomic_add [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
 define amdgpu_kernel void @atomic_add_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile add i32 addrspace(1)* %out, i32 %in seq_cst
@@ -86,9 +112,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_addr64:
+; GCN-LABEL: {{^}}atomic_add_i32_addr64:
 ; SI: buffer_atomic_add v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_add v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -96,10 +123,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_add_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_add_i32_ret_addr64:
 ; SI: buffer_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_add [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_add_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -108,8 +137,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_offset:
-; GCN: buffer_atomic_and v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_and_i32_offset:
+; SIVI: buffer_atomic_and v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_and_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -117,9 +148,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_offset:
-; GCN: buffer_atomic_and [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_and_i32_ret_offset:
+; SIVI: buffer_atomic_and [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_and_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -128,9 +161,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_and_i32_addr64_offset:
 ; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_and_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -139,10 +174,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_and_i32_ret_addr64_offset:
 ; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_and [[RET:v[0-9]]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_and [[RET:v[0-9]]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -152,17 +189,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32:
-; GCN: buffer_atomic_and v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_and_i32:
+; SIVI: buffer_atomic_and v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_and v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_and_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_ret:
-; GCN: buffer_atomic_and [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_and_i32_ret:
+; SIVI: buffer_atomic_and [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_and_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile and i32 addrspace(1)* %out, i32 %in seq_cst
@@ -170,9 +211,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_addr64:
+; GCN-LABEL: {{^}}atomic_and_i32_addr64:
 ; SI: buffer_atomic_and v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_and v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_and_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -180,10 +223,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_and_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_and_i32_ret_addr64:
 ; SI: buffer_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_and [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_and_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -192,8 +237,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_offset:
-; GCN: buffer_atomic_sub v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_sub_i32_offset:
+; SIVI: buffer_atomic_sub v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_sub_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -201,9 +248,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_offset:
-; GCN: buffer_atomic_sub [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_sub_i32_ret_offset:
+; SIVI: buffer_atomic_sub [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_sub v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_sub_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -212,9 +261,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_sub_i32_addr64_offset:
 ; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_sub_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -223,10 +274,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_sub_i32_ret_addr64_offset:
 ; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -236,17 +289,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32:
-; GCN: buffer_atomic_sub v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_sub_i32:
+; SIVI: buffer_atomic_sub v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_sub v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_sub_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret:
-; GCN: buffer_atomic_sub [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_sub_i32_ret:
+; SIVI: buffer_atomic_sub [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_sub_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile sub i32 addrspace(1)* %out, i32 %in seq_cst
@@ -254,9 +311,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_addr64:
+; GCN-LABEL: {{^}}atomic_sub_i32_addr64:
 ; SI: buffer_atomic_sub v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_sub v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_sub v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_sub_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -264,10 +323,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_sub_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_sub_i32_ret_addr64:
 ; SI: buffer_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_sub [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_sub_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -276,8 +337,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_offset:
-; GCN: buffer_atomic_smax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_max_i32_offset:
+; SIVI: buffer_atomic_smax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_max_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -285,9 +348,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_offset:
-; GCN: buffer_atomic_smax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_max_i32_ret_offset:
+; SIVI: buffer_atomic_smax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_max_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -296,9 +361,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_max_i32_addr64_offset:
 ; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_max_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -307,10 +374,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_max_i32_ret_addr64_offset:
 ; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -320,17 +389,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32:
-; GCN: buffer_atomic_smax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_max_i32:
+; SIVI: buffer_atomic_smax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_max_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_ret:
-; GCN: buffer_atomic_smax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_max_i32_ret:
+; SIVI: buffer_atomic_smax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_max_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile max i32 addrspace(1)* %out, i32 %in seq_cst
@@ -338,9 +411,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_addr64:
+; GCN-LABEL: {{^}}atomic_max_i32_addr64:
 ; SI: buffer_atomic_smax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+
+; GFX9: global_atomic_smax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_max_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -348,10 +423,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_max_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_max_i32_ret_addr64:
 ; SI: buffer_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_max_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -360,8 +437,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_offset:
-; GCN: buffer_atomic_umax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_umax_i32_offset:
+; SIVI: buffer_atomic_umax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_umax_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -369,9 +448,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_offset:
-; GCN: buffer_atomic_umax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_umax_i32_ret_offset:
+; SIVI: buffer_atomic_umax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_umax_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -380,9 +461,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_umax_i32_addr64_offset:
 ; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_umax_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -391,10 +473,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_umax_i32_ret_addr64_offset:
 ; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -404,17 +488,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32:
-; GCN: buffer_atomic_umax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_umax_i32:
+; SIVI: buffer_atomic_umax v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_umax_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret:
-; GCN: buffer_atomic_umax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_umax_i32_ret:
+; SIVI: buffer_atomic_umax [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umax_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile umax i32 addrspace(1)* %out, i32 %in seq_cst
@@ -422,9 +510,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_addr64:
+; GCN-LABEL: {{^}}atomic_umax_i32_addr64:
 ; SI: buffer_atomic_umax v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_umax v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_umax_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -432,10 +521,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umax_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_umax_i32_ret_addr64:
 ; SI: buffer_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umax [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umax_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -444,8 +535,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_offset:
-; GCN: buffer_atomic_smin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_min_i32_offset:
+; SIVI: buffer_atomic_smin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_min_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -453,9 +546,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_offset:
-; GCN: buffer_atomic_smin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_min_i32_ret_offset:
+; SIVI: buffer_atomic_smin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_min_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -464,9 +559,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_min_i32_addr64_offset:
 ; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16
 define amdgpu_kernel void @atomic_min_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -475,10 +571,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_min_i32_ret_addr64_offset:
 ; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -488,17 +586,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32:
-; GCN: buffer_atomic_smin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_min_i32:
+; SIVI: buffer_atomic_smin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_min_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_ret:
-; GCN: buffer_atomic_smin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_min_i32_ret:
+; SIVI: buffer_atomic_smin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_min_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile min i32 addrspace(1)* %out, i32 %in seq_cst
@@ -506,9 +608,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_addr64:
+; GCN-LABEL: {{^}}atomic_min_i32_addr64:
 ; SI: buffer_atomic_smin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_smin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_min_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -516,10 +619,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_min_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_min_i32_ret_addr64:
 ; SI: buffer_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_smin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_min_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -528,8 +633,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_offset:
-; GCN: buffer_atomic_umin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_umin_i32_offset:
+; SIVI: buffer_atomic_umin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_umin_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -537,9 +644,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_offset:
-; GCN: buffer_atomic_umin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_umin_i32_ret_offset:
+; SIVI: buffer_atomic_umin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_umin_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -548,9 +657,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_umin_i32_addr64_offset:
 ; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_umin_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -559,10 +669,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_umin_i32_ret_addr64_offset:
 ; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -572,17 +684,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32:
-; GCN: buffer_atomic_umin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_umin_i32:
+; SIVI: buffer_atomic_umin v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_umin_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret:
-; SI: buffer_atomic_umin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_umin_i32_ret:
+; SIVI: buffer_atomic_umin [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umin_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile umin i32 addrspace(1)* %out, i32 %in seq_cst
@@ -590,9 +705,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_addr64:
+; GCN-LABEL: {{^}}atomic_umin_i32_addr64:
 ; SI: buffer_atomic_umin v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_umin v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_umin_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -600,10 +716,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_umin_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_umin_i32_ret_addr64:
 ; SI: buffer_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_umin [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umin_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -612,8 +730,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_offset:
-; GCN: buffer_atomic_or v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_or_i32_offset:
+; SIVI: buffer_atomic_or v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_or_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -621,9 +741,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_offset:
-; GCN: buffer_atomic_or [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_or_i32_ret_offset:
+; SIVI: buffer_atomic_or [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_or_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -632,9 +754,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_or_i32_addr64_offset:
 ; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16
 define amdgpu_kernel void @atomic_or_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -643,10 +766,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_or_i32_ret_addr64_offset:
 ; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -656,17 +781,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32:
-; GCN: buffer_atomic_or v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_or_i32:
+; SIVI: buffer_atomic_or v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_or_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_ret:
-; GCN: buffer_atomic_or [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_or_i32_ret:
+; SIVI: buffer_atomic_or [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_or_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile or i32 addrspace(1)* %out, i32 %in seq_cst
@@ -674,9 +803,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_addr64:
+; GCN-LABEL: {{^}}atomic_or_i32_addr64:
 ; SI: buffer_atomic_or v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_or v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_or_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -684,10 +814,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_or_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_or_i32_ret_addr64:
 ; SI: buffer_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_or [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_or_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -696,8 +828,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_offset:
-; GCN: buffer_atomic_swap v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_xchg_i32_offset:
+; SIVI: buffer_atomic_swap v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -705,9 +839,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_offset:
-; GCN: buffer_atomic_swap [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_xchg_i32_ret_offset:
+; SIVI: buffer_atomic_swap [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -716,10 +852,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_xchg_i32_addr64_offset:
 ; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
-
-; VI: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}{{$}}
+; VI: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -728,11 +864,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_xchg_i32_ret_addr64_offset:
 ; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-
 ; VI: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -742,17 +879,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32:
-; GCN: buffer_atomic_swap v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_xchg_i32:
+; SIVI: buffer_atomic_swap v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_xchg_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret:
-; GCN: buffer_atomic_swap [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_xchg_i32_ret:
+; SIVI: buffer_atomic_swap [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
@@ -760,9 +900,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_addr64:
+; GCN-LABEL: {{^}}atomic_xchg_i32_addr64:
 ; SI: buffer_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -770,10 +911,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xchg_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_xchg_i32_ret_addr64:
 ; SI: buffer_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_swap [[RET:v[0-9]+]],  v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_swap [[RET:v[0-9]+]],  v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -782,8 +925,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_offset:
-; GCN: buffer_atomic_cmpswap v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_offset:
+; SIVI: buffer_atomic_cmpswap v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:16{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_offset(i32 addrspace(1)* %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -791,9 +936,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_ret_offset:
-; GCN: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]{{:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword v[[RET]]
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_offset:
+; SIVI: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]{{:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword v[[RET]]
+
+; GFX9: global_atomic_cmpswap [[RET:v[0-9]+]],  v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -803,10 +950,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_addr64_offset:
 ; SI: buffer_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 
 ; VI: flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX9: global_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], off offset:16{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index, i32 %old) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -815,10 +963,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64_offset:
 ; SI: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dword v[[RET]]
+; SIVI: buffer_store_dword v[[RET]]
+
+; GFX9: global_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index, i32 %old) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -829,17 +979,21 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32:
-; GCN: buffer_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32:
+; SIVI: buffer_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+
+; GFX9: global_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32(i32 addrspace(1)* %out, i32 %in, i32 %old) {
 entry:
   %val = cmpxchg volatile i32 addrspace(1)* %out, i32 %old, i32 %in seq_cst seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_ret:
-; GCN: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword v[[RET]]
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret:
+; SIVI: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword v[[RET]]
+
+; GFX9: global_atomic_cmpswap [[RET:v[0-9]+]], v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i32 %old) {
 entry:
   %val = cmpxchg volatile i32 addrspace(1)* %out, i32 %old, i32 %in seq_cst seq_cst
@@ -848,9 +1002,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_addr64:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_addr64:
 ; SI: buffer_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]{{$}}
+; GFX9: global_atomic_cmpswap v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index, i32 %old) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -858,10 +1013,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i32_ret_addr64:
 ; SI: buffer_atomic_cmpswap v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dword v[[RET]]
+; SIVI: buffer_store_dword v[[RET]]
+
+; GFX9: global_atomic_cmpswap v[[RET:[0-9]+]], v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index, i32 %old) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -871,8 +1028,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_offset:
-; GCN: buffer_atomic_xor v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; GCN-LABEL: {{^}}atomic_xor_i32_offset:
+; SIVI: buffer_atomic_xor v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_xor v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_xor_i32_offset(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -880,9 +1039,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_offset:
-; GCN: buffer_atomic_xor [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_xor_i32_ret_offset:
+; SIVI: buffer_atomic_xor [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_xor v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_xor_i32_ret_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -891,9 +1052,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_xor_i32_addr64_offset:
 ; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
 ; VI: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_xor_i32_addr64_offset(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -902,10 +1064,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_xor_i32_ret_addr64_offset:
 ; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -915,17 +1079,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32:
-; GCN: buffer_atomic_xor v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_xor_i32:
+; SIVI: buffer_atomic_xor v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_xor_i32(i32 addrspace(1)* %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret:
-; GCN: buffer_atomic_xor [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dword [[RET]]
+; GCN-LABEL: {{^}}atomic_xor_i32_ret:
+; SIVI: buffer_atomic_xor [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xor_i32_ret(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in) {
 entry:
   %val = atomicrmw volatile xor i32 addrspace(1)* %out, i32 %in seq_cst
@@ -933,9 +1100,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_addr64:
+; GCN-LABEL: {{^}}atomic_xor_i32_addr64:
 ; SI: buffer_atomic_xor v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: global_atomic_xor v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_xor_i32_addr64(i32 addrspace(1)* %out, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -943,10 +1111,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_xor_i32_ret_addr64:
+; GCN-LABEL: {{^}}atomic_xor_i32_ret_addr64:
 ; SI: buffer_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_atomic_xor [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xor_i32_ret_addr64(i32 addrspace(1)* %out, i32 addrspace(1)* %out2, i32 %in, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -955,10 +1125,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i32_offset:
+; GCN-LABEL: {{^}}atomic_load_i32_offset:
 ; SI: buffer_load_dword [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
 ; VI: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_load_i32_offset(i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %in, i64 4
@@ -967,10 +1139,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i32:
+; GCN-LABEL: {{^}}atomic_load_i32:
 ; SI: buffer_load_dword [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
 ; VI: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], off glc
 define amdgpu_kernel void @atomic_load_i32(i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
 entry:
   %val = load atomic i32, i32 addrspace(1)* %in seq_cst, align 4
@@ -978,10 +1152,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i32_addr64_offset:
+; GCN-LABEL: {{^}}atomic_load_i32_addr64_offset:
 ; SI: buffer_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
 ; VI: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], off offset:16 glc{{$}}
 define amdgpu_kernel void @atomic_load_i32_addr64_offset(i32 addrspace(1)* %in, i32 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %in, i64 %index
@@ -991,10 +1167,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i32_addr64:
+; GCN-LABEL: {{^}}atomic_load_i32_addr64:
 ; SI: buffer_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dword [[RET]]
+; SIVI: buffer_store_dword [[RET]]
+
+; GFX9: global_load_dword [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_load_i32_addr64(i32 addrspace(1)* %in, i32 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %in, i64 %index
@@ -1003,9 +1181,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i32_offset:
-; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
-; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i32_offset:
+; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, i32 addrspace(1)* %out) {
 entry:
   %gep = getelementptr i32, i32 addrspace(1)* %out, i64 4
@@ -1013,18 +1192,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i32:
-; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc{{$}}
-; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i32:
+; SI: buffer_store_dword {{v[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_store_i32(i32 %in, i32 addrspace(1)* %out) {
 entry:
   store atomic i32 %in, i32 addrspace(1)* %out seq_cst, align 4
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i32_addr64_offset:
-; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16 glc{{$}}
-; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i32_addr64_offset:
+; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:16{{$}}
+; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}, off offset:16{{$}}
 define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, i32 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
@@ -1033,9 +1214,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i32_addr64:
-; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}} glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i32_addr64:
+; SI: buffer_store_dword {{v[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}, off{{$}}
 define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, i32 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i32, i32 addrspace(1)* %out, i64 %index
diff --git a/test/CodeGen/AMDGPU/global_atomics_i64.ll b/test/CodeGen/AMDGPU/global_atomics_i64.ll
index 56520b787ead7..e63b5a9de2210 100644
--- a/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -1,8 +1,11 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; GCN-LABEL: {{^}}atomic_add_i64_offset:
-; GCN: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+
+; GFX9: global_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], off offset:32{{$}}
 define amdgpu_kernel void @atomic_add_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -11,8 +14,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_add_i64_ret_offset:
-; GCN: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_add_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -24,6 +29,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_add_i64_addr64_offset:
 ; CI: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}{{$}}
+; GFX9: global_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_add_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -35,7 +41,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_add_i64_ret_addr64_offset:
 ; CI: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -46,7 +54,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_add_i64:
-; GCN: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; SIVI: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile add i64 addrspace(1)* %out, i64 %in seq_cst
@@ -54,8 +63,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_add_i64_ret:
-; GCN: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_add_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile add i64 addrspace(1)* %out, i64 %in seq_cst
@@ -66,6 +77,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_add_i64_addr64:
 ; CI: buffer_atomic_add_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_add_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_add_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -76,7 +88,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_add_i64_ret_addr64:
 ; CI: buffer_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_add_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_add_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -86,7 +100,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_and_i64_offset:
-; GCN: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_and_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -95,8 +110,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_and_i64_ret_offset:
-; GCN: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_and_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -108,6 +125,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_and_i64_addr64_offset:
 ; CI: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_and_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -119,7 +137,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_and_i64_ret_addr64_offset:
 ; CI: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -130,7 +150,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_and_i64:
-; GCN: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_and_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile and i64 addrspace(1)* %out, i64 %in seq_cst
@@ -138,8 +159,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_and_i64_ret:
-; GCN: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_and_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile and i64 addrspace(1)* %out, i64 %in seq_cst
@@ -150,6 +173,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_and_i64_addr64:
 ; CI: buffer_atomic_and_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_and_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_and_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -160,7 +184,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_and_i64_ret_addr64:
 ; CI: buffer_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_and_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_and_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -170,7 +196,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_sub_i64_offset:
-; GCN: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_sub_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -179,8 +206,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_sub_i64_ret_offset:
-; GCN: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_sub_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -192,6 +221,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_sub_i64_addr64_offset:
 ; CI: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_sub_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -203,7 +233,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_sub_i64_ret_addr64_offset:
 ; CI: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -214,7 +246,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_sub_i64:
-; GCN: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_sub_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile sub i64 addrspace(1)* %out, i64 %in seq_cst
@@ -222,8 +255,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_sub_i64_ret:
-; GCN: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_sub_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile sub i64 addrspace(1)* %out, i64 %in seq_cst
@@ -234,6 +269,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_sub_i64_addr64:
 ; CI: buffer_atomic_sub_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_sub_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_sub_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -244,7 +280,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_sub_i64_ret_addr64:
 ; CI: buffer_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_sub_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_sub_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -254,7 +292,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_max_i64_offset:
-; GCN: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_max_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -263,8 +302,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_max_i64_ret_offset:
-; GCN: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_max_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -276,6 +317,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_max_i64_addr64_offset:
 ; CI: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_max_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -287,7 +329,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_max_i64_ret_addr64_offset:
 ; CI: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -298,7 +342,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_max_i64:
-; GCN: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_max_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile max i64 addrspace(1)* %out, i64 %in seq_cst
@@ -306,8 +351,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_max_i64_ret:
-; GCN: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_max_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile max i64 addrspace(1)* %out, i64 %in seq_cst
@@ -318,6 +365,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_max_i64_addr64:
 ; CI: buffer_atomic_smax_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_smax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_max_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -328,7 +376,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_max_i64_ret_addr64:
 ; CI: buffer_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_max_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -338,7 +388,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umax_i64_offset:
-; GCN: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_umax_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -347,8 +398,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umax_i64_ret_offset:
-; GCN: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_umax_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -360,6 +413,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umax_i64_addr64_offset:
 ; CI: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; FX9: global_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} offset:32{{$}}
 define amdgpu_kernel void @atomic_umax_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -371,7 +425,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umax_i64_ret_addr64_offset:
 ; CI: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -382,7 +438,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umax_i64:
-; GCN: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_umax_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile umax i64 addrspace(1)* %out, i64 %in seq_cst
@@ -390,8 +447,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umax_i64_ret:
-; GCN: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umax_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile umax i64 addrspace(1)* %out, i64 %in seq_cst
@@ -402,6 +461,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umax_i64_addr64:
 ; CI: buffer_atomic_umax_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_umax_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_umax_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -412,7 +472,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umax_i64_ret_addr64:
 ; CI: buffer_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umax_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umax_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -422,7 +484,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_min_i64_offset:
-; GCN: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_min_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -431,8 +494,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_min_i64_ret_offset:
-; GCN: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_min_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -444,6 +509,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_min_i64_addr64_offset:
 ; CI: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_min_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -455,7 +521,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_min_i64_ret_addr64_offset:
 ; CI: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -466,7 +534,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_min_i64:
-; GCN: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_min_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile min i64 addrspace(1)* %out, i64 %in seq_cst
@@ -474,8 +543,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_min_i64_ret:
-; GCN: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_min_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile min i64 addrspace(1)* %out, i64 %in seq_cst
@@ -486,6 +557,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_min_i64_addr64:
 ; CI: buffer_atomic_smin_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_smin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_min_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -496,7 +568,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_min_i64_ret_addr64:
 ; CI: buffer_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_smin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_min_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -506,7 +580,9 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umin_i64_offset:
-; GCN: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+
+; GFX9: global_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_umin_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -515,8 +591,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umin_i64_ret_offset:
-; GCN: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_umin_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -528,6 +606,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umin_i64_addr64_offset:
 ; CI: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_umin_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -539,7 +618,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umin_i64_ret_addr64_offset:
 ; CI: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -550,7 +631,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umin_i64:
-; GCN: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_umin_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile umin i64 addrspace(1)* %out, i64 %in seq_cst
@@ -558,8 +640,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_umin_i64_ret:
-; CI: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umin_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile umin i64 addrspace(1)* %out, i64 %in seq_cst
@@ -570,6 +654,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umin_i64_addr64:
 ; CI: buffer_atomic_umin_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_umin_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_umin_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -580,7 +665,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_umin_i64_ret_addr64:
 ; CI: buffer_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_umin_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_umin_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -590,7 +677,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_or_i64_offset:
-; GCN: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_or_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -599,8 +687,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_or_i64_ret_offset:
-; GCN: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_or_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -612,6 +702,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_or_i64_addr64_offset:
 ; CI: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_or_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -623,7 +714,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_or_i64_ret_addr64_offset:
 ; CI: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -634,7 +727,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_or_i64:
-; GCN: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_or_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile or i64 addrspace(1)* %out, i64 %in seq_cst
@@ -642,8 +736,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_or_i64_ret:
-; GCN: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_or_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile or i64 addrspace(1)* %out, i64 %in seq_cst
@@ -654,6 +750,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_or_i64_addr64:
 ; CI: buffer_atomic_or_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_or_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_or_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -664,7 +761,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_or_i64_ret_addr64:
 ; CI: buffer_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_or_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_or_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -674,7 +773,9 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xchg_i64_offset:
-; GCN: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+
+; GFX9: global_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -683,8 +784,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset:
-; GCN: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -696,6 +799,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xchg_i64_addr64_offset:
 ; CI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}{{$}}
+; GFX9: global_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -707,7 +811,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret_addr64_offset:
 ; CI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -718,7 +824,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xchg_i64:
-; GCN: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_xchg_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile xchg i64 addrspace(1)* %out, i64 %in seq_cst
@@ -726,8 +833,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret:
-; GCN: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile xchg i64 addrspace(1)* %out, i64 %in seq_cst
@@ -738,6 +847,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xchg_i64_addr64:
 ; CI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -748,7 +858,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret_addr64:
 ; CI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]],  v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -758,7 +870,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xor_i64_offset:
-; GCN: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; CIVI: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_xor_i64_offset(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -767,8 +880,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xor_i64_ret_offset:
-; GCN: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_xor_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -780,6 +895,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xor_i64_addr64_offset:
 ; CI: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
 ; VI: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_xor_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -791,7 +907,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xor_i64_ret_addr64_offset:
 ; CI: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -802,7 +920,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xor_i64:
-; GCN: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; CIVI: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_xor_i64(i64 addrspace(1)* %out, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile xor i64 addrspace(1)* %out, i64 %in seq_cst
@@ -810,8 +929,10 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}atomic_xor_i64_ret:
-; GCN: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xor_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in) {
 entry:
   %tmp0 = atomicrmw volatile xor i64 addrspace(1)* %out, i64 %in seq_cst
@@ -822,6 +943,7 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xor_i64_addr64:
 ; CI: buffer_atomic_xor_x2 v{{\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+; GFX9: global_atomic_xor_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_xor_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -832,7 +954,9 @@ entry:
 ; GCN-LABEL: {{^}}atomic_xor_i64_ret_addr64:
 ; CI: buffer_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_atomic_xor_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off glc{{$}}
 define amdgpu_kernel void @atomic_xor_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -842,15 +966,9 @@ entry:
 }
 
 
-
-
-
-
-
-
-
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_offset:
-; GCN: buffer_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_offset:
+; CIVI: buffer_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_offset(i64 addrspace(1)* %out, i64 %in, i64 %old) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -858,9 +976,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_soffset:
-; GCN: s_mov_b32 [[SREG:s[0-9]+]], 0x11940
-; GCN: buffer_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_soffset:
+; CIVI: s_mov_b32 [[SREG:s[0-9]+]], 0x11940
+; CIVI: buffer_atomic_cmpswap_x2 v[{{[0-9]+}}:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], [[SREG]]{{$}}
+
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(i64 addrspace(1)* %out, i64 %in, i64 %old) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 9000
@@ -868,9 +988,11 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_ret_offset:
-; GCN: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]{{:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; GCN: buffer_store_dwordx2 v{{\[}}[[RET]]:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_offset:
+; CIVI: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]{{:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
+; CIVI: buffer_store_dwordx2 v{{\[}}[[RET]]:
+
+; GFX9: global_atomic_cmpswap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %old) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -880,10 +1002,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_addr64_offset:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_addr64_offset:
 ; CI: buffer_atomic_cmpswap_x2 v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
-
 ; VI: flat_atomic_cmpswap_x2 v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], off offset:32{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(i64 addrspace(1)* %out, i64 %in, i64 %index, i64 %old) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -892,10 +1014,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64_offset:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64_offset:
 ; CI: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dwordx2 v{{\[}}[[RET]]:
+; CIVI: buffer_store_dwordx2 v{{\[}}[[RET]]:
+
+; GFX9: global_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index, i64 %old) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -906,17 +1030,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64:
-; GCN: buffer_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64:
+; CIVI: buffer_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64(i64 addrspace(1)* %out, i64 %in, i64 %old) {
 entry:
   %val = cmpxchg volatile i64 addrspace(1)* %out, i64 %old, i64 %in seq_cst seq_cst
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_ret:
-; GCN: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; GCN: buffer_store_dwordx2 v{{\[}}[[RET]]:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret:
+; CIVI: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
+; CIVI: buffer_store_dwordx2 v{{\[}}[[RET]]:
+
+; GFX9: global_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %old) {
 entry:
   %val = cmpxchg volatile i64 addrspace(1)* %out, i64 %old, i64 %in seq_cst seq_cst
@@ -925,9 +1052,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_addr64:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_addr64:
 ; CI: buffer_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
 ; VI: flat_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}]{{$}}
+; GFX9: global_atomic_cmpswap_x2 v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(i64 addrspace(1)* %out, i64 %in, i64 %index, i64 %old) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -935,10 +1063,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64:
+; GCN-LABEL: {{^}}atomic_cmpxchg_i64_ret_addr64:
 ; CI: buffer_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dwordx2 v{{\[}}[[RET]]:
+; CIVI: buffer_store_dwordx2 v{{\[}}[[RET]]:
+
+; GFX9: global_atomic_cmpswap_x2 v{{\[}}[[RET:[0-9]+]]:{{[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], v[{{[0-9]+:[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(i64 addrspace(1)* %out, i64 addrspace(1)* %out2, i64 %in, i64 %index, i64 %old) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -948,10 +1078,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i64_offset:
+; GCN-LABEL: {{^}}atomic_load_i64_offset:
 ; CI: buffer_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
 ; VI: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}], off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_load_i64_offset(i64 addrspace(1)* %in, i64 addrspace(1)* %out) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %in, i64 4
@@ -960,10 +1092,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i64:
+; GCN-LABEL: {{^}}atomic_load_i64:
 ; CI: buffer_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
 ; VI: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_load_i64(i64 addrspace(1)* %in, i64 addrspace(1)* %out) {
 entry:
   %val = load atomic i64, i64 addrspace(1)* %in seq_cst, align 8
@@ -971,10 +1105,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i64_addr64_offset:
+; GCN-LABEL: {{^}}atomic_load_i64_addr64_offset:
 ; CI: buffer_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
 ; VI: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], off offset:32 glc{{$}}
 define amdgpu_kernel void @atomic_load_i64_addr64_offset(i64 addrspace(1)* %in, i64 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %in, i64 %index
@@ -984,10 +1120,12 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_load_i64_addr64:
+; GCN-LABEL: {{^}}atomic_load_i64_addr64:
 ; CI: buffer_load_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
 ; VI: flat_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}] glc{{$}}
-; GCN: buffer_store_dwordx2 [[RET]]
+; CIVI: buffer_store_dwordx2 [[RET]]
+
+; GFX9: global_load_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], off glc{{$}}
 define amdgpu_kernel void @atomic_load_i64_addr64(i64 addrspace(1)* %in, i64 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %in, i64 %index
@@ -996,9 +1134,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i64_offset:
-; CI: buffer_store_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
-; VI: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i64_offset:
+; CI: buffer_store_dwordx2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+; VI: flat_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX9: global_store_dwordx2 [[RET:v\[[0-9]+:[0-9]\]]], v[{{[0-9]+}}:{{[0-9]+}}], off offset:32{{$}}
 define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, i64 addrspace(1)* %out) {
 entry:
   %gep = getelementptr i64, i64 addrspace(1)* %out, i64 4
@@ -1006,18 +1145,20 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i64:
-; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 glc
-; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}] glc
+; GCN-LABEL: {{^}}atomic_store_i64:
+; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0{{$}}
+; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GFX9: global_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_store_i64(i64 %in, i64 addrspace(1)* %out) {
 entry:
   store atomic i64 %in, i64 addrspace(1)* %out seq_cst, align 8
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i64_addr64_offset:
-; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32 glc{{$}}
-; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i64_addr64_offset:
+; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 offset:32{{$}}
+; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}}
+; GFX9: global_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], off offset:32{{$}}
 define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, i64 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
@@ -1026,9 +1167,10 @@ entry:
   ret void
 }
 
-; FUNC-LABEL: {{^}}atomic_store_i64_addr64:
-; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64 glc{{$}}
-; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}] glc{{$}}
+; GCN-LABEL: {{^}}atomic_store_i64_addr64:
+; CI: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]\]}}, v[{{[0-9]+}}:{{[0-9]+}}], s[{{[0-9]+}}:{{[0-9]+}}], 0 addr64{{$}}
+; VI: flat_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}]{{$}}
+; GFX9: global_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v[{{[0-9]+:[0-9]+}}], off{{$}}
 define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, i64 addrspace(1)* %out, i64 %index) {
 entry:
   %ptr = getelementptr i64, i64 addrspace(1)* %out, i64 %index
diff --git a/test/CodeGen/AMDGPU/half.ll b/test/CodeGen/AMDGPU/half.ll
index 43745d4b3da3d..0df58519a6217 100644
--- a/test/CodeGen/AMDGPU/half.ll
+++ b/test/CodeGen/AMDGPU/half.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; half args should be promoted to float for SI and lower.
 
diff --git a/test/CodeGen/AMDGPU/hazard-inlineasm.mir b/test/CodeGen/AMDGPU/hazard-inlineasm.mir
new file mode 100644
index 0000000000000..6c03227d3715a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/hazard-inlineasm.mir
@@ -0,0 +1,24 @@
+# RUN: llc -mcpu=gfx900 -march=amdgcn -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck %s
+
+# If an INLINEASM statement is preceded by a vmem store of more than 8 bytes *and*
+# the INLINEASM defs the vregs holding the data-to-be-stored by that preceding store,
+# then the hazard recognizer should insert a s_nop in between them.
+
+...
+
+# GCN-LABEL: name: hazard-inlineasm
+# CHECK: FLAT_STORE_DWORDX4
+# CHECK-NEXT: S_NOP 0
+# CHECK-NEXT: INLINEASM
+
+---
+name: hazard-inlineasm
+
+body: |
+  bb.0:
+   FLAT_STORE_DWORDX4 %vgpr49_vgpr50, %vgpr26_vgpr27_vgpr28_vgpr29, 0, 0, 0, implicit %exec, implicit %flat_scr
+   INLINEASM $"v_mad_u64_u32 $0, $1, $2, $3, $4", 0, 2621450, def %vgpr26_vgpr27, 2818058, def dead %sgpr14_sgpr15, 589833, %sgpr12, 327689, killed %vgpr51, 2621449, %vgpr46_vgpr47
+   S_ENDPGM
+...
+
+
diff --git a/test/CodeGen/AMDGPU/hazard.mir b/test/CodeGen/AMDGPU/hazard.mir
index d495a327e9e39..80afcbf197aa4 100644
--- a/test/CodeGen/AMDGPU/hazard.mir
+++ b/test/CodeGen/AMDGPU/hazard.mir
@@ -1,6 +1,7 @@
 # RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN -check-prefix=VI %s
 # RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
+# GCN-LABEL: name: hazard_implicit_def
 # GCN:    bb.0.entry:
 # GCN:      %m0 = S_MOV_B32
 # GFX9:     S_NOP 0
@@ -29,3 +30,31 @@ body:             |
     SI_RETURN_TO_EPILOG killed %vgpr5, killed %vgpr0
 
 ...
+
+# GCN-LABEL: name: hazard_inlineasm
+# GCN:    bb.0.entry:
+# GCN:      %m0 = S_MOV_B32
+# GFX9:     S_NOP 0
+# VI-NOT:   S_NOP_0
+# GCN:      V_INTERP_P1_F32
+---
+name:            hazard_inlineasm
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+liveins:
+  - { reg: '%sgpr7', virtual-reg: '' }
+  - { reg: '%vgpr4', virtual-reg: '' }
+body:             |
+  bb.0.entry:
+    liveins: %sgpr7, %vgpr4
+
+    %m0 = S_MOV_B32 killed %sgpr7
+    INLINEASM $"; no-op", 1, 327690, def %vgpr5
+    %vgpr0 = V_INTERP_P1_F32 killed %vgpr4, 0, 0, implicit %m0, implicit %exec
+    SI_RETURN_TO_EPILOG killed %vgpr5, killed %vgpr0
+...
diff --git a/test/CodeGen/AMDGPU/hsa-func.ll b/test/CodeGen/AMDGPU/hsa-func.ll
index 35aeeeaa225ce..0bf1c6a23c311 100644
--- a/test/CodeGen/AMDGPU/hsa-func.ll
+++ b/test/CodeGen/AMDGPU/hsa-func.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-flat-for-global | FileCheck --check-prefix=HSA-CI %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA-CI %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo  | FileCheck --check-prefix=HSA %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-flat-for-global | FileCheck --check-prefix=HSA-VI %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo | FileCheck --check-prefix=HSA-VI %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri -filetype=obj | llvm-readobj -symbols -s -sd | FileCheck --check-prefix=ELF %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | llvm-mc -filetype=obj -triple amdgcn--amdhsa -mcpu=kaveri | llvm-readobj -symbols -s -sd | FileCheck %s --check-prefix=ELF
 
@@ -27,7 +27,7 @@
 
 ; ELF: Symbol {
 ; ELF: Name: simple
-; ELF: Size: 48
+; ELF: Size: 36
 ; ELF: Type: Function (0x2)
 ; ELF: }
 
@@ -41,12 +41,9 @@
 ; HSA: .p2align 2
 ; HSA: {{^}}simple:
 ; HSA-NOT: amd_kernel_code_t
-; HSA-NOT: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x0
+; HSA: flat_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v[0:1]
 
 ; Make sure we are setting the ATC bit:
-; HSA-CI: s_mov_b32 s[[HI:[0-9]+]], 0x100f000
-; On VI+ we also need to set MTYPE = 2
-; HSA-VI: s_mov_b32 s[[HI:[0-9]+]], 0x1100f000
 ; Make sure we generate flat store for HSA
 ; HSA: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
 
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-deduce-ro-arg.ll b/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
index a33c3646e253e..76545b0377e72 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-deduce-ro-arg.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-deduce-ro-arg.ll
@@ -1,24 +1,26 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck %s
 
 ; CHECK:      - Name:            test_ro_arg
-; CHECK:        Args:
-; CHECK-NEXT: - Size:            8
+; CHECK-NEXT:   SymbolName:      'test_ro_arg@kd'
+; CHECK-NEXT:   Args:
+; CHECK-NEXT: - Name:            in
+; CHECK-NEXT:   TypeName:        'float*'
+; CHECK-NEXT:   Size:            8
 ; CHECK-NEXT:   Align:           8
 ; CHECK-NEXT:   ValueKind:       GlobalBuffer
 ; CHECK-NEXT:   ValueType:       F32
-; CHECK-NEXT:   AccQual:         ReadOnly
 ; CHECK-NEXT:   AddrSpaceQual:   Global
+; CHECK-NEXT:   AccQual:         ReadOnly
 ; CHECK-NEXT:   IsConst:         true
 ; CHECK-NEXT:   IsRestrict:      true
+; CHECK-NEXT: - Name:            out
 ; CHECK-NEXT:   TypeName:        'float*'
-
-; CHECK-NEXT: - Size:            8
+; CHECK-NEXT:   Size:            8
 ; CHECK-NEXT:   Align:           8
 ; CHECK-NEXT:   ValueKind:       GlobalBuffer
 ; CHECK-NEXT:   ValueType:       F32
-; CHECK-NEXT:   AccQual:         Default
 ; CHECK-NEXT:   AddrSpaceQual:   Global
-; CHECK-NEXT:   TypeName:        'float*'
+; CHECK-NEXT:   AccQual:         Default
 
 define amdgpu_kernel void @test_ro_arg(float addrspace(1)* noalias readonly %in, float addrspace(1)* %out)
     !kernel_arg_addr_space !0 !kernel_arg_access_qual !1 !kernel_arg_type !2
@@ -30,4 +32,3 @@ define amdgpu_kernel void @test_ro_arg(float addrspace(1)* noalias readonly %in,
 !1 = !{!"none", !"none"}
 !2 = !{!"float*", !"float*"}
 !3 = !{!"const restrict", !""}
-
diff --git a/test/CodeGen/AMDGPU/hsa-metadata-enqueu-kernel.ll b/test/CodeGen/AMDGPU/hsa-metadata-enqueu-kernel.ll
new file mode 100644
index 0000000000000..83baf5be57b1b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/hsa-metadata-enqueu-kernel.ll
@@ -0,0 +1,98 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+
+; CHECK: ---
+; CHECK:  Version: [ 1, 0 ]
+; CHECK-NOT:  Printf:
+; CHECK:  Kernels:
+
+; CHECK:      - Name:            test_non_enqueue_kernel_caller
+; CHECK-NEXT:   SymbolName:      'test_non_enqueue_kernel_caller@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NOT:        ValueKind:     HiddenNone
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
+define amdgpu_kernel void @test_non_enqueue_kernel_caller(i8 %a)
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:      - Name:            test_enqueue_kernel_caller
+; CHECK-NEXT:   SymbolName:      'test_enqueue_kernel_caller@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenNone
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenDefaultQueue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenCompletionAction
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3
+    !kernel_arg_base_type !3 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+attributes #0 = { "calls-enqueue-kernel" }
+
+!1 = !{i32 0}
+!2 = !{!"none"}
+!3 = !{!"char"}
+!4 = !{!""}
+
+!opencl.ocl.version = !{!90}
+!90 = !{i32 2, i32 0}
+
+
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-from-llvm-ir-full.ll b/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
index 37fd08242fbaa..06fc5eae06169 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-from-llvm-ir-full.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-from-llvm-ir-full.ll
@@ -1,9 +1,9 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-dump-comd -amdgpu-verify-comd -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -amdgpu-dump-comd -amdgpu-verify-comd -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-comd -amdgpu-verify-comd -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -amdgpu-dump-hsa-metadata -amdgpu-verify-hsa-metadata -filetype=obj -o - < %s 2>&1 | FileCheck --check-prefix=PARSER %s
 
 %struct.A = type { i8, float }
 %opencl.image1d_t = type opaque
@@ -14,6 +14,8 @@
 %struct.B = type { i32 addrspace(1)*}
 %opencl.clk_event_t = type opaque
 
+@__test_block_invoke_kernel_runtime_handle = external addrspace(1) externally_initialized constant i8 addrspace(1)*
+
 ; CHECK: ---
 ; CHECK:  Version: [ 1, 0 ]
 ; CHECK:  Printf:
@@ -22,15 +24,17 @@
 ; CHECK:  Kernels:
 
 ; CHECK:      - Name:            test_char
+; CHECK-NEXT:   SymbolName:      'test_char@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          1
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
 ; CHECK-NEXT:       Align:         1
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      char
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -48,6 +52,8 @@
 ; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NOT:        ValueKind:     HiddenDefaultQueue
+; CHECK-NOT:        ValueKind:     HiddenCompletionAction
 define amdgpu_kernel void @test_char(i8 %a)
     !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
     !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
@@ -55,15 +61,17 @@ define amdgpu_kernel void @test_char(i8 %a)
 }
 
 ; CHECK:      - Name:            test_ushort2
+; CHECK-NEXT:   SymbolName:      'test_ushort2@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      ushort2
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     U16
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      ushort2
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -88,15 +96,17 @@ define amdgpu_kernel void @test_ushort2(<2 x i16> %a)
 }
 
 ; CHECK:      - Name:            test_int3
+; CHECK-NEXT:   SymbolName:      'test_int3@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          16
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int3
+; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int3
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -121,15 +131,17 @@ define amdgpu_kernel void @test_int3(<3 x i32> %a)
 }
 
 ; CHECK:      - Name:            test_ulong4
+; CHECK-NEXT:   SymbolName:      'test_ulong4@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          32
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      ulong4
+; CHECK-NEXT:       Size:          32
 ; CHECK-NEXT:       Align:         32
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     U64
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      ulong4
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -154,15 +166,17 @@ define amdgpu_kernel void @test_ulong4(<4 x i64> %a)
 }
 
 ; CHECK:      - Name:            test_half8
+; CHECK-NEXT:   SymbolName:      'test_half8@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          16
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      half8
+; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     F16
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      half8
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -187,15 +201,17 @@ define amdgpu_kernel void @test_half8(<8 x half> %a)
 }
 
 ; CHECK:      - Name:            test_float16
+; CHECK-NEXT:   SymbolName:      'test_float16@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          64
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      float16
+; CHECK-NEXT:       Size:          64
 ; CHECK-NEXT:       Align:         64
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     F32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      float16
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -220,15 +236,17 @@ define amdgpu_kernel void @test_float16(<16 x float> %a)
 }
 
 ; CHECK:      - Name:            test_double16
+; CHECK-NEXT:   SymbolName:      'test_double16@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          128
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      double16
+; CHECK-NEXT:       Size:          128
 ; CHECK-NEXT:       Align:         128
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     F64
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      double16
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -253,16 +271,18 @@ define amdgpu_kernel void @test_double16(<16 x double> %a)
 }
 
 ; CHECK:      - Name:            test_pointer
+; CHECK-NEXT:   SymbolName:      'test_pointer@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      'int *'
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      'int *'
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -287,16 +307,18 @@ define amdgpu_kernel void @test_pointer(i32 addrspace(1)* %a)
 }
 
 ; CHECK:      - Name:            test_image
+; CHECK-NEXT:   SymbolName:      'test_image@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      image2d_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      image2d_t
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -321,15 +343,17 @@ define amdgpu_kernel void @test_image(%opencl.image2d_t addrspace(1)* %a)
 }
 
 ; CHECK:      - Name:            test_sampler
+; CHECK-NEXT:   SymbolName:      'test_sampler@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      sampler_t
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     Sampler
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      sampler_t
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -354,16 +378,18 @@ define amdgpu_kernel void @test_sampler(i32 %a)
 }
 
 ; CHECK:      - Name:            test_queue
+; CHECK-NEXT:   SymbolName:      'test_queue@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      queue_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Queue
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      queue_t
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -388,16 +414,18 @@ define amdgpu_kernel void @test_queue(%opencl.queue_t addrspace(1)* %a)
 }
 
 ; CHECK:      - Name:            test_struct
+; CHECK-NEXT:   SymbolName:      'test_struct@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      struct A
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Private
-; CHECK-NEXT:       TypeName:      struct A
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -422,15 +450,17 @@ define amdgpu_kernel void @test_struct(%struct.A* byval %a)
 }
 
 ; CHECK:      - Name:            test_i128
+; CHECK-NEXT:   SymbolName:      'test_i128@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          16
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      i128
+; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     Struct
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      i128
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -455,27 +485,31 @@ define amdgpu_kernel void @test_i128(i128 %a)
 }
 
 ; CHECK:      - Name:            test_multi_arg
+; CHECK-NEXT:   SymbolName:      'test_multi_arg@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          b
+; CHECK-NEXT:       TypeName:      short2
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I16
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      short2
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          c
+; CHECK-NEXT:       TypeName:      char3
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      char3
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -500,31 +534,35 @@ define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c)
 }
 
 ; CHECK:      - Name:            test_addr_space
+; CHECK-NEXT:   SymbolName:      'test_addr_space@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          g
+; CHECK-NEXT:       TypeName:      'int *'
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          c
 ; CHECK-NEXT:       TypeName:      'int *'
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Constant
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          l
 ; CHECK-NEXT:       TypeName:      'int *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       PointeeAlign:  4
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'int *'
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -551,34 +589,38 @@ define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g,
 }
 
 ; CHECK:      - Name:            test_type_qual
+; CHECK-NEXT:   SymbolName:      'test_type_qual@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      'int *'
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsVolatile:    true
+; CHECK-NEXT:     - Name:          b
 ; CHECK-NEXT:       TypeName:      'int *'
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsConst:       true
 ; CHECK-NEXT:       IsRestrict:    true
+; CHECK-NEXT:     - Name:          c
 ; CHECK-NEXT:       TypeName:      'int *'
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Pipe
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       IsPipe:        true
-; CHECK-NEXT:       TypeName:      'int *'
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -605,30 +647,34 @@ define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a,
 }
 
 ; CHECK:      - Name:            test_access_qual
+; CHECK-NEXT:   SymbolName:      'test_access_qual@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          ro
+; CHECK-NEXT:       TypeName:      image1d_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       ReadOnly
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      image1d_t
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       AccQual:       ReadOnly
+; CHECK-NEXT:     - Name:          wo
+; CHECK-NEXT:       TypeName:      image2d_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       WriteOnly
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      image2d_t
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       AccQual:       WriteOnly
+; CHECK-NEXT:     - Name:          rw
+; CHECK-NEXT:       TypeName:      image3d_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     Image
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       ReadWrite
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      image3d_t
+; CHECK-NEXT:       AccQual:       ReadWrite
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -655,17 +701,19 @@ define amdgpu_kernel void @test_access_qual(%opencl.image1d_t addrspace(1)* %ro,
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_half
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_half@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   half
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -690,17 +738,19 @@ define amdgpu_kernel void @test_vec_type_hint_half(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_float
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_float@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   float
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -725,17 +775,19 @@ define amdgpu_kernel void @test_vec_type_hint_float(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_double
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_double@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   double
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -760,17 +812,19 @@ define amdgpu_kernel void @test_vec_type_hint_double(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_char
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_char@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   char
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -795,17 +849,19 @@ define amdgpu_kernel void @test_vec_type_hint_char(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_short
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_short@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   short
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -830,17 +886,19 @@ define amdgpu_kernel void @test_vec_type_hint_short(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_long
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_long@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   long
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -865,17 +923,19 @@ define amdgpu_kernel void @test_vec_type_hint_long(i32 %a)
 }
 
 ; CHECK:      - Name:            test_vec_type_hint_unknown
+; CHECK-NEXT:   SymbolName:      'test_vec_type_hint_unknown@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       VecTypeHint:   unknown
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      int
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      int
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -900,18 +960,20 @@ define amdgpu_kernel void @test_vec_type_hint_unknown(i32 %a)
 }
 
 ; CHECK:      - Name:            test_reqd_wgs_vec_type_hint
+; CHECK-NEXT:   SymbolName:      'test_reqd_wgs_vec_type_hint@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       ReqdWorkGroupSize: [ 1, 2, 4 ]
 ; CHECK-NEXT:       VecTypeHint:       int
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:              4
+; CHECK-NEXT:     - Name:              a
+; CHECK-NEXT:       TypeName:          int
+; CHECK-NEXT:       Size:              4
 ; CHECK-NEXT:       Align:             4
 ; CHECK-NEXT:       ValueKind:         ByValue
 ; CHECK-NEXT:       ValueType:         I32
 ; CHECK-NEXT:       AccQual:           Default
-; CHECK-NEXT:       TypeName:          int
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetX
@@ -937,18 +999,20 @@ define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a)
 }
 
 ; CHECK:      - Name:            test_wgs_hint_vec_type_hint
+; CHECK-NEXT:   SymbolName:      'test_wgs_hint_vec_type_hint@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Attrs:
 ; CHECK-NEXT:       WorkGroupSizeHint: [ 8, 16, 32 ]
 ; CHECK-NEXT:       VecTypeHint:       uint4
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:              4
+; CHECK-NEXT:     - Name:              a
+; CHECK-NEXT:       TypeName:          int
+; CHECK-NEXT:       Size:              4
 ; CHECK-NEXT:       Align:             4
 ; CHECK-NEXT:       ValueKind:         ByValue
 ; CHECK-NEXT:       ValueType:         I32
 ; CHECK-NEXT:       AccQual:           Default
-; CHECK-NEXT:       TypeName:          int
 ; CHECK-NEXT:     - Size:              8
 ; CHECK-NEXT:       Align:             8
 ; CHECK-NEXT:       ValueKind:         HiddenGlobalOffsetX
@@ -974,16 +1038,18 @@ define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a)
 }
 
 ; CHECK:      - Name:            test_arg_ptr_to_ptr
+; CHECK-NEXT:   SymbolName:      'test_arg_ptr_to_ptr@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      'int **'
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I32
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      'int **'
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -1008,16 +1074,18 @@ define amdgpu_kernel void @test_arg_ptr_to_ptr(i32* addrspace(1)* %a)
 }
 
 ; CHECK:      - Name:            test_arg_struct_contains_ptr
+; CHECK-NEXT:   SymbolName:      'test_arg_struct_contains_ptr@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      struct B
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Private
-; CHECK-NEXT:       TypeName:      struct B
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -1042,15 +1110,17 @@ define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B* byval %a)
 }
 
 ; CHECK:      - Name:            test_arg_vector_of_ptr
+; CHECK-NEXT:   SymbolName:      'test_arg_vector_of_ptr@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          16
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      'global int* __attribute__((ext_vector_type(2)))'
+; CHECK-NEXT:       Size:          16
 ; CHECK-NEXT:       Align:         16
 ; CHECK-NEXT:       ValueKind:     ByValue
 ; CHECK-NEXT:       ValueType:     I32
 ; CHECK-NEXT:       AccQual:       Default
-; CHECK-NEXT:       TypeName:      'global int* __attribute__((ext_vector_type(2)))'
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -1075,16 +1145,18 @@ define amdgpu_kernel void @test_arg_vector_of_ptr(<2 x i32 addrspace(1)*> %a)
 }
 
 ; CHECK:      - Name:            test_arg_unknown_builtin_type
+; CHECK-NEXT:   SymbolName:      'test_arg_unknown_builtin_type@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      clk_event_t
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     Struct
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      clk_event_t
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -1110,64 +1182,72 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(
 }
 
 ; CHECK:      - Name:            test_pointee_align
+; CHECK-NEXT:   SymbolName:      'test_pointee_align@kd'
 ; CHECK-NEXT:   Language:        OpenCL C
 ; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
 ; CHECK-NEXT:   Args:
-; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      'long *'
+; CHECK-NEXT:       Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     GlobalBuffer
 ; CHECK-NEXT:       ValueType:     I64
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Global
-; CHECK-NEXT:       TypeName:      'long *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          b
+; CHECK-NEXT:       TypeName:      'char *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  1
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          c
+; CHECK-NEXT:       TypeName:      'char2 *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  2
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char2 *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          d
+; CHECK-NEXT:       TypeName:      'char3 *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  4
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char3 *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          e
+; CHECK-NEXT:       TypeName:      'char4 *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  4
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char4 *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          f
+; CHECK-NEXT:       TypeName:      'char8 *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  8
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char8 *'
-; CHECK-NEXT:     - Size:          4
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Name:          g
+; CHECK-NEXT:       TypeName:      'char16 *'
+; CHECK-NEXT:       Size:          4
 ; CHECK-NEXT:       Align:         4
 ; CHECK-NEXT:       ValueKind:     DynamicSharedPointer
 ; CHECK-NEXT:       ValueType:     I8
 ; CHECK-NEXT:       PointeeAlign:  16
-; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:       AddrSpaceQual: Local
-; CHECK-NEXT:       TypeName:      'char16 *'
+; CHECK-NEXT:       AccQual:       Default
 ; CHECK-NEXT:     - Size:          8
 ; CHECK-NEXT:       Align:         8
 ; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
@@ -1197,6 +1277,92 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
   ret void
 }
 
+; CHECK:      - Name:            __test_block_invoke_kernel
+; CHECK-NEXT:   SymbolName:      '__test_block_invoke_kernel@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Attrs:
+; CHECK-NEXT:       RuntimeHandle: __test_block_invoke_kernel_runtime_handle
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          arg
+; CHECK-NEXT:       TypeName:      __block_literal
+; CHECK-NEXT:       Size:          25
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     Struct
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @__test_block_invoke_kernel(
+    <{ i32, i32, i8 addrspace(4)*, i8 addrspace(1)*, i8 }> %arg) #0
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !110
+    !kernel_arg_base_type !110 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+; CHECK:      - Name:            test_enqueue_kernel_caller
+; CHECK-NEXT:   SymbolName:      'test_enqueue_kernel_caller@kd'
+; CHECK-NEXT:   Language:        OpenCL C
+; CHECK-NEXT:   LanguageVersion: [ 2, 0 ]
+; CHECK-NEXT:   Args:
+; CHECK-NEXT:     - Name:          a
+; CHECK-NEXT:       TypeName:      char
+; CHECK-NEXT:       Size:          1
+; CHECK-NEXT:       Align:         1
+; CHECK-NEXT:       ValueKind:     ByValue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AccQual:       Default
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetX
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetY
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenGlobalOffsetZ
+; CHECK-NEXT:       ValueType:     I64
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenPrintfBuffer
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenDefaultQueue
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+; CHECK-NEXT:     - Size:          8
+; CHECK-NEXT:       Align:         8
+; CHECK-NEXT:       ValueKind:     HiddenCompletionAction
+; CHECK-NEXT:       ValueType:     I8
+; CHECK-NEXT:       AddrSpaceQual: Global
+define amdgpu_kernel void @test_enqueue_kernel_caller(i8 %a) #1
+    !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9
+    !kernel_arg_base_type !9 !kernel_arg_type_qual !4 {
+  ret void
+}
+
+attributes #0 = { "runtime-handle"="__test_block_invoke_kernel_runtime_handle" }
+attributes #1 = { "calls-enqueue-kernel" }
+
 !llvm.printf.fmts = !{!100, !101}
 
 !1 = !{i32 0}
@@ -1250,13 +1416,6 @@ define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a,
 !94 = !{!"", !"", !"", !"", !"", !"", !""}
 !100 = !{!"1:1:4:%d\5Cn"}
 !101 = !{!"2:1:8:%g\5Cn"}
+!110 = !{!"__block_literal"}
 
-; NOTES: Displaying notes found at file offset 0x{{[0-9]+}}
-; NOTES-NEXT: Owner    Data size    Description
-; NOTES-NEXT: AMD      0x00000008   Unknown note type: (0x00000001)
-; NOTES-NEXT: AMD      0x0000001b   Unknown note type: (0x00000003)
-; GFX700:     AMD      0x00008b0a   Unknown note type: (0x0000000a)
-; GFX800:     AMD      0x00008e6e   Unknown note type: (0x0000000a)
-; GFX900:     AMD      0x00008b0a   Unknown note type: (0x0000000a)
-
-; PARSER: AMDGPU Code Object Metadata Parser Test: PASS
+; PARSER: AMDGPU HSA Metadata Parser Test: PASS
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-images.ll b/test/CodeGen/AMDGPU/hsa-metadata-images.ll
index 918560469852b..f6290d41f6905 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-images.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-images.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
 
 %opencl.image1d_t = type opaque
 %opencl.image1d_array_t = type opaque
@@ -19,44 +19,57 @@
 ; CHECK:  Version: [ 1, 0 ]
 
 ; CHECK:  Kernels:
-; CHECK:    - Name: test
+; CHECK:    - Name:       test
+; CHECK:      SymbolName: 'test@kd'
 ; CHECK:      Args:
-; CHECK:        - Size:      8
-; CHECK:          ValueKind: Image
+; CHECK:        - Name:      a
 ; CHECK:          TypeName:  image1d_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      b
 ; CHECK:          TypeName:  image1d_array_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      c
 ; CHECK:          TypeName:  image1d_buffer_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      d
 ; CHECK:          TypeName:  image2d_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      e
 ; CHECK:          TypeName:  image2d_array_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      f
 ; CHECK:          TypeName:  image2d_array_depth_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      g
 ; CHECK:          TypeName:  image2d_array_msaa_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      h
 ; CHECK:          TypeName:  image2d_array_msaa_depth_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      i
 ; CHECK:          TypeName:  image2d_depth_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      j
 ; CHECK:          TypeName:  image2d_msaa_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      k
 ; CHECK:          TypeName:  image2d_msaa_depth_t
-; CHECK:        - Size:      8
+; CHECK:          Size:      8
 ; CHECK:          ValueKind: Image
+; CHECK:        - Name:      l
 ; CHECK:          TypeName:  image3d_t
+; CHECK:          Size:      8
+; CHECK:          ValueKind: Image
 define amdgpu_kernel void @test(%opencl.image1d_t addrspace(1)* %a,
                                 %opencl.image1d_array_t addrspace(1)* %b,
                                 %opencl.image1d_buffer_t addrspace(1)* %c,
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-1.ll b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll
index f41da9f921361..f46b07d80b5d1 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-1.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-2.ll b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll
index 0509663d9849a..f4e584b5ae399 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-2.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-3.ll b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll
index 7404cec5d78ac..5e951dd3f8f7e 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-invalid-ocl-version-3.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-invalid-ocl-version-3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck %s
 
 ; Make sure llc does not crash for invalid opencl version metadata.
 
diff --git a/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll b/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
new file mode 100644
index 0000000000000..f4a914adddbf7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props.ll
@@ -0,0 +1,142 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX803 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+
+@var = addrspace(1) global float 0.0
+
+; CHECK: ---
+; CHECK:  Version: [ 1, 0 ]
+; CHECK:  Kernels:
+
+; CHECK: - Name:       test
+; CHECK:   SymbolName: 'test@kd'
+; CHECK:   CodeProps:
+; CHECK:     KernargSegmentSize:      24
+; CHECK:     GroupSegmentFixedSize:   0
+; CHECK:     PrivateSegmentFixedSize: 0
+; CHECK:     KernargSegmentAlign:     8
+; CHECK:     WavefrontSize:           64
+; CHECK:     NumSGPRs:                6
+; GFX700:    NumVGPRs:                4
+; GFX803:    NumVGPRs:                6
+; GFX900:    NumVGPRs:                6
+; CHECK:     MaxFlatWorkGroupSize:    256
+define amdgpu_kernel void @test(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a,
+    half addrspace(1)* %b) {
+entry:
+  %a.val = load half, half addrspace(1)* %a
+  %b.val = load half, half addrspace(1)* %b
+  %r.val = fadd half %a.val, %b.val
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}
+
+; CHECK: - Name:       num_spilled_sgprs
+; CHECK:   SymbolName: 'num_spilled_sgprs@kd'
+; CHECK:   CodeProps:
+; CHECK:     NumSpilledSGPRs: 41
+define amdgpu_kernel void @num_spilled_sgprs(
+    i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 addrspace(1)* %out2,
+    i32 addrspace(1)* %out3, i32 addrspace(1)* %out4, i32 addrspace(1)* %out5,
+    i32 addrspace(1)* %out6, i32 addrspace(1)* %out7, i32 addrspace(1)* %out8,
+    i32 addrspace(1)* %out9, i32 addrspace(1)* %outa, i32 addrspace(1)* %outb,
+    i32 addrspace(1)* %outc, i32 addrspace(1)* %outd, i32 addrspace(1)* %oute,
+    i32 addrspace(1)* %outf, i32 %in0, i32 %in1, i32 %in2, i32 %in3, i32 %in4,
+    i32 %in5, i32 %in6, i32 %in7, i32 %in8, i32 %in9, i32 %ina, i32 %inb,
+    i32 %inc, i32 %ind, i32 %ine, i32 %inf) #0 {
+entry:
+  store i32 %in0, i32 addrspace(1)* %out0
+  store i32 %in1, i32 addrspace(1)* %out1
+  store i32 %in2, i32 addrspace(1)* %out2
+  store i32 %in3, i32 addrspace(1)* %out3
+  store i32 %in4, i32 addrspace(1)* %out4
+  store i32 %in5, i32 addrspace(1)* %out5
+  store i32 %in6, i32 addrspace(1)* %out6
+  store i32 %in7, i32 addrspace(1)* %out7
+  store i32 %in8, i32 addrspace(1)* %out8
+  store i32 %in9, i32 addrspace(1)* %out9
+  store i32 %ina, i32 addrspace(1)* %outa
+  store i32 %inb, i32 addrspace(1)* %outb
+  store i32 %inc, i32 addrspace(1)* %outc
+  store i32 %ind, i32 addrspace(1)* %outd
+  store i32 %ine, i32 addrspace(1)* %oute
+  store i32 %inf, i32 addrspace(1)* %outf
+  ret void
+}
+
+; CHECK: - Name:       num_spilled_vgprs
+; CHECK:   SymbolName: 'num_spilled_vgprs@kd'
+; CHECK:   CodeProps:
+; CHECK:     NumSpilledVGPRs: 14
+define amdgpu_kernel void @num_spilled_vgprs() #1 {
+  %val0 = load volatile float, float addrspace(1)* @var
+  %val1 = load volatile float, float addrspace(1)* @var
+  %val2 = load volatile float, float addrspace(1)* @var
+  %val3 = load volatile float, float addrspace(1)* @var
+  %val4 = load volatile float, float addrspace(1)* @var
+  %val5 = load volatile float, float addrspace(1)* @var
+  %val6 = load volatile float, float addrspace(1)* @var
+  %val7 = load volatile float, float addrspace(1)* @var
+  %val8 = load volatile float, float addrspace(1)* @var
+  %val9 = load volatile float, float addrspace(1)* @var
+  %val10 = load volatile float, float addrspace(1)* @var
+  %val11 = load volatile float, float addrspace(1)* @var
+  %val12 = load volatile float, float addrspace(1)* @var
+  %val13 = load volatile float, float addrspace(1)* @var
+  %val14 = load volatile float, float addrspace(1)* @var
+  %val15 = load volatile float, float addrspace(1)* @var
+  %val16 = load volatile float, float addrspace(1)* @var
+  %val17 = load volatile float, float addrspace(1)* @var
+  %val18 = load volatile float, float addrspace(1)* @var
+  %val19 = load volatile float, float addrspace(1)* @var
+  %val20 = load volatile float, float addrspace(1)* @var
+  %val21 = load volatile float, float addrspace(1)* @var
+  %val22 = load volatile float, float addrspace(1)* @var
+  %val23 = load volatile float, float addrspace(1)* @var
+  %val24 = load volatile float, float addrspace(1)* @var
+  %val25 = load volatile float, float addrspace(1)* @var
+  %val26 = load volatile float, float addrspace(1)* @var
+  %val27 = load volatile float, float addrspace(1)* @var
+  %val28 = load volatile float, float addrspace(1)* @var
+  %val29 = load volatile float, float addrspace(1)* @var
+  %val30 = load volatile float, float addrspace(1)* @var
+
+  store volatile float %val0, float addrspace(1)* @var
+  store volatile float %val1, float addrspace(1)* @var
+  store volatile float %val2, float addrspace(1)* @var
+  store volatile float %val3, float addrspace(1)* @var
+  store volatile float %val4, float addrspace(1)* @var
+  store volatile float %val5, float addrspace(1)* @var
+  store volatile float %val6, float addrspace(1)* @var
+  store volatile float %val7, float addrspace(1)* @var
+  store volatile float %val8, float addrspace(1)* @var
+  store volatile float %val9, float addrspace(1)* @var
+  store volatile float %val10, float addrspace(1)* @var
+  store volatile float %val11, float addrspace(1)* @var
+  store volatile float %val12, float addrspace(1)* @var
+  store volatile float %val13, float addrspace(1)* @var
+  store volatile float %val14, float addrspace(1)* @var
+  store volatile float %val15, float addrspace(1)* @var
+  store volatile float %val16, float addrspace(1)* @var
+  store volatile float %val17, float addrspace(1)* @var
+  store volatile float %val18, float addrspace(1)* @var
+  store volatile float %val19, float addrspace(1)* @var
+  store volatile float %val20, float addrspace(1)* @var
+  store volatile float %val21, float addrspace(1)* @var
+  store volatile float %val22, float addrspace(1)* @var
+  store volatile float %val23, float addrspace(1)* @var
+  store volatile float %val24, float addrspace(1)* @var
+  store volatile float %val25, float addrspace(1)* @var
+  store volatile float %val26, float addrspace(1)* @var
+  store volatile float %val27, float addrspace(1)* @var
+  store volatile float %val28, float addrspace(1)* @var
+  store volatile float %val29, float addrspace(1)* @var
+  store volatile float %val30, float addrspace(1)* @var
+
+  ret void
+}
+
+attributes #0 = { "amdgpu-num-sgpr"="14" }
+attributes #1 = { "amdgpu-num-vgpr"="20" }
diff --git a/test/CodeGen/AMDGPU/code-object-metadata-kernel-debug-props.ll b/test/CodeGen/AMDGPU/hsa-metadata-kernel-debug-props.ll
index 0ffc922031539..80d033d3c1b0c 100644
--- a/test/CodeGen/AMDGPU/code-object-metadata-kernel-debug-props.ll
+++ b/test/CodeGen/AMDGPU/hsa-metadata-kernel-debug-props.ll
@@ -1,6 +1,7 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -amdgpu-code-object-metadata -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=gfx700 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX700 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=gfx800 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX800 --check-prefix=NOTES %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=gfx900 -filetype=obj -o - < %s | llvm-readobj -elf-output-style=GNU -notes | FileCheck --check-prefix=CHECK --check-prefix=GFX900 --check-prefix=NOTES %s
+target datalayout = "A5"
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
@@ -8,27 +9,28 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK:  Version: [ 1, 0 ]
 
 ; CHECK:  Kernels:
-; CHECK:    - Name: test
+; CHECK:    - Name:       test
+; CHECK:      SymbolName: 'test@kd'
 ; CHECK:      DebugProps:
 ; CHECK:        DebuggerABIVersion:                [ 1, 0 ]
 ; CHECK:        ReservedNumVGPRs:                  4
 ; GFX700:       ReservedFirstVGPR:                 8
 ; GFX800:       ReservedFirstVGPR:                 8
-; GFX9:         ReservedFirstVGPR:                 14
+; GFX900:       ReservedFirstVGPR:                 10
 ; CHECK:        PrivateSegmentBufferSGPR:          0
 ; CHECK:        WavefrontPrivateSegmentOffsetSGPR: 11
 define amdgpu_kernel void @test(i32 addrspace(1)* %A) #0 !dbg !7 !kernel_arg_addr_space !12 !kernel_arg_access_qual !13 !kernel_arg_type !14 !kernel_arg_base_type !14 !kernel_arg_type_qual !15 {
 entry:
-  %A.addr = alloca i32 addrspace(1)*, align 4
-  store i32 addrspace(1)* %A, i32 addrspace(1)** %A.addr, align 4
-  call void @llvm.dbg.declare(metadata i32 addrspace(1)** %A.addr, metadata !16, metadata !17), !dbg !18
-  %0 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !19
+  %A.addr = alloca i32 addrspace(1)*, align 4, addrspace(5)
+  store i32 addrspace(1)* %A, i32 addrspace(1)* addrspace(5)* %A.addr, align 4
+  call void @llvm.dbg.declare(metadata i32 addrspace(1)* addrspace(5)* %A.addr, metadata !16, metadata !17), !dbg !18
+  %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !19
   %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %0, i64 0, !dbg !19
   store i32 777, i32 addrspace(1)* %arrayidx, align 4, !dbg !20
-  %1 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !21
+  %1 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !21
   %arrayidx1 = getelementptr inbounds i32, i32 addrspace(1)* %1, i64 1, !dbg !21
   store i32 888, i32 addrspace(1)* %arrayidx1, align 4, !dbg !22
-  %2 = load i32 addrspace(1)*, i32 addrspace(1)** %A.addr, align 4, !dbg !23
+  %2 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(5)* %A.addr, align 4, !dbg !23
   %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %2, i64 2, !dbg !23
   store i32 999, i32 addrspace(1)* %arrayidx2, align 4, !dbg !24
   ret void, !dbg !25
@@ -55,7 +57,7 @@ attributes #0 = { noinline nounwind "correctly-rounded-divide-sqrt-fp-math"="fal
 !11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !12 = !{i32 1}
 !13 = !{!"none"}
-!14 = !{!"int*"}
+!14 = !{!"int addrspace(5)*"}
 !15 = !{!""}
 !16 = !DILocalVariable(name: "A", arg: 1, scope: !7, file: !1, line: 1, type: !10)
 !17 = !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)
diff --git a/test/CodeGen/AMDGPU/hsa-note-no-func.ll b/test/CodeGen/AMDGPU/hsa-note-no-func.ll
index 81d9ed2eba8cc..bfe8737b4ff26 100644
--- a/test/CodeGen/AMDGPU/hsa-note-no-func.ll
+++ b/test/CodeGen/AMDGPU/hsa-note-no-func.ll
@@ -4,6 +4,8 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx701 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI701 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx702 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI702 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx703 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx704 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI704 %s
+; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=bonaire | FileCheck --check-prefix=HSA --check-prefix=HSA-CI704 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=mullins | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=hawaii | FileCheck --check-prefix=HSA --check-prefix=HSA-CI701 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kabini | FileCheck --check-prefix=HSA --check-prefix=HSA-CI703 %s
@@ -17,12 +19,9 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx801 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI801 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx802 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI802 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx803 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx804 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI804 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx810 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI810 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx900 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX900 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx901 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX901 %s
 ; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx902 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX902 %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx903 | FileCheck --check-prefix=HSA --check-prefix=HSA-GFX903 %s
 
 ; HSA: .hsa_code_object_version 2,1
 ; HSA-SI600: .hsa_code_object_isa 6,0,0,"AMD","AMDGPU"
@@ -31,13 +30,11 @@
 ; HSA-CI701: .hsa_code_object_isa 7,0,1,"AMD","AMDGPU"
 ; HSA-CI702: .hsa_code_object_isa 7,0,2,"AMD","AMDGPU"
 ; HSA-CI703: .hsa_code_object_isa 7,0,3,"AMD","AMDGPU"
+; HSA-CI704: .hsa_code_object_isa 7,0,4,"AMD","AMDGPU"
 ; HSA-VI800: .hsa_code_object_isa 8,0,0,"AMD","AMDGPU"
 ; HSA-VI801: .hsa_code_object_isa 8,0,1,"AMD","AMDGPU"
 ; HSA-VI802: .hsa_code_object_isa 8,0,2,"AMD","AMDGPU"
 ; HSA-VI803: .hsa_code_object_isa 8,0,3,"AMD","AMDGPU"
-; HSA-VI804: .hsa_code_object_isa 8,0,4,"AMD","AMDGPU"
 ; HSA-VI810: .hsa_code_object_isa 8,1,0,"AMD","AMDGPU"
 ; HSA-GFX900: .hsa_code_object_isa 9,0,0,"AMD","AMDGPU"
-; HSA-GFX901: .hsa_code_object_isa 9,0,1,"AMD","AMDGPU"
 ; HSA-GFX902: .hsa_code_object_isa 9,0,2,"AMD","AMDGPU"
-; HSA-GFX903: .hsa_code_object_isa 9,0,3,"AMD","AMDGPU"
diff --git a/test/CodeGen/AMDGPU/huge-private-buffer.ll b/test/CodeGen/AMDGPU/huge-private-buffer.ll
new file mode 100644
index 0000000000000..7e2d0b0ec9a0c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/huge-private-buffer.ll
@@ -0,0 +1,31 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}scratch_buffer_known_high_bit_small:
+; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 4
+; GCN-NOT: [[FI]]
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[FI]]
+define amdgpu_kernel void @scratch_buffer_known_high_bit_small() #0 {
+  %alloca = alloca i32, align 4
+  store volatile i32 0, i32* %alloca
+  %toint = ptrtoint i32* %alloca to i32
+  %masked = and i32 %toint, 2147483647
+  store volatile i32 %masked, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}scratch_buffer_known_high_bit_huge:
+; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 4
+; GCN-DAG: buffer_store_dword
+; GCN-DAG: v_and_b32_e32 [[MASKED:v[0-9]+]], 0x7ffffffc, [[FI]]
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[MASKED]]
+define amdgpu_kernel void @scratch_buffer_known_high_bit_huge() #1 {
+  %alloca = alloca i32, align 4
+  store volatile i32 0, i32* %alloca
+  %toint = ptrtoint i32* %alloca to i32
+  %masked = and i32 %toint, 2147483647
+  store volatile i32 %masked, i32 addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "target-features"="+huge-private-buffer" }
diff --git a/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll b/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll
index f6bf0b09486e8..37d05c7ac414c 100644
--- a/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll
+++ b/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll
@@ -3,7 +3,7 @@
 
 ; SILowerI1Copies was not handling IMPLICIT_DEF
 ; SI-LABEL: {{^}}br_implicit_def:
-; SI: BB#0:
+; SI: %bb.0:
 ; SI-NEXT: s_cbranch_scc1
 define amdgpu_kernel void @br_implicit_def(i32 addrspace(1)* %out, i32 %arg) #0 {
 bb:
diff --git a/test/CodeGen/AMDGPU/i1-copy-phi.ll b/test/CodeGen/AMDGPU/i1-copy-phi.ll
index 4a0213dd1de55..63a9f1feb6dff 100644
--- a/test/CodeGen/AMDGPU/i1-copy-phi.ll
+++ b/test/CodeGen/AMDGPU/i1-copy-phi.ll
@@ -4,11 +4,9 @@
 ; SI-LABEL: {{^}}br_i1_phi:
 ; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}}
 ; SI: s_and_saveexec_b64
-; SI: s_xor_b64
 ; SI: v_mov_b32_e32 [[REG]], -1{{$}}
 ; SI: v_cmp_ne_u32_e32 vcc, 0, [[REG]]
 ; SI: s_and_saveexec_b64
-; SI: s_xor_b64
 ; SI: s_endpgm
 define amdgpu_kernel void @br_i1_phi(i32 %arg) {
 bb:
diff --git a/test/CodeGen/AMDGPU/imm.ll b/test/CodeGen/AMDGPU/imm.ll
index 8cda01a10f765..4e20d038fa710 100644
--- a/test/CodeGen/AMDGPU/imm.ll
+++ b/test/CodeGen/AMDGPU/imm.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 ; Use a 64-bit value with lo bits that can be represented as an inline constant
 ; GCN-LABEL: {{^}}i64_imm_inline_lo:
@@ -129,7 +129,7 @@ define amdgpu_kernel void @store_inline_imm_inv_2pi_f32(float addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}store_inline_imm_m_inv_2pi_f32:
-; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xbe22f983{{$}}
+; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0xbe22f983{{$}}
 ; GCN: buffer_store_dword [[REG]]
 define amdgpu_kernel void @store_inline_imm_m_inv_2pi_f32(float addrspace(1)* %out) {
   store float 0xBFC45F3060000000, float addrspace(1)* %out
diff --git a/test/CodeGen/AMDGPU/immv216.ll b/test/CodeGen/AMDGPU/immv216.ll
index fe86a58729681..1135ab03ca3c8 100644
--- a/test/CodeGen/AMDGPU/immv216.ll
+++ b/test/CodeGen/AMDGPU/immv216.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
 ; FIXME: Merge into imm.ll
@@ -282,9 +282,9 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_v2f16(<2 x half> addrspace(1)*
 ; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 0.5
 ; GFX9: buffer_store_dword [[REG]]
 
-; VI: v_mov_b32_e32 [[CONST05:v[0-9]+]], 0x3800
 ; VI: buffer_load_dword
 ; VI-NOT: and
+; VI: v_mov_b32_e32 [[CONST05:v[0-9]+]], 0x3800
 ; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, [[CONST05]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; VI-DAG: v_add_f16_e32 v{{[0-9]+}}, 0.5, v{{[0-9]+}}
 ; VI: v_or_b32
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll b/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
index 8e207a38c8474..63384f5e44508 100644
--- a/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s
 
 ; FIXME: Merge into indirect-addressing-si.ll
 
@@ -17,3 +17,48 @@ entry:
   ret void
 }
 
+; Make sure we don't hit use of undefined register errors when expanding an
+; extract with undef index.
+
+; CHECK-LABEL: {{^}}extract_adjacent_blocks:
+; CHECK: s_load_dword [[ARG:s[0-9]+]]
+; CHECK: s_cmp_lg_u32
+; CHECK: s_cbranch_scc1 [[BB4:BB[0-9]+_[0-9]+]]
+
+; CHECK: buffer_load_dwordx4
+; CHECK: s_mov_b32 m0,
+; CHECK: v_movrels_b32_e32
+
+; CHECK: s_branch [[ENDBB:BB[0-9]+_[0-9]+]]
+
+; CHECK: [[BB4]]:
+; CHECK: buffer_load_dwordx4
+; CHECK: s_mov_b32 m0,
+; CHECK: v_movrels_b32_e32
+
+; CHECK: [[ENDBB]]:
+; CHECK: buffer_store_dword
+; CHECK: s_endpgm
+
+define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) #0 {
+bb:
+  %tmp = icmp eq i32 %arg, 0
+  br i1 %tmp, label %bb1, label %bb4
+
+bb1:
+  %tmp2 = load volatile <4 x float>, <4 x float> addrspace(1)* undef
+  %tmp3 = extractelement <4 x float> %tmp2, i32 undef
+  call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp2) #0 ; Prevent block optimize out
+  br label %bb7
+
+bb4:
+  %tmp5 = load volatile <4 x float>, <4 x float> addrspace(1)* undef
+  %tmp6 = extractelement <4 x float> %tmp5, i32 undef
+  call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp5) #0 ; Prevent block optimize out
+  br label %bb7
+
+bb7:
+  %tmp8 = phi float [ %tmp3, %bb1 ], [ %tmp6, %bb4 ]
+  store volatile float %tmp8, float addrspace(1)* undef
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 62200b988bea3..2b4aca019a391 100644
--- a/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -1,7 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-vgpr-index-mode -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=IDXMODE %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=IDXMODE %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL,PREGFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MOVREL,PREGFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-vgpr-index-mode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,IDXMODE,PREGFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,IDXMODE,GFX9 %s
 
 ; Tests for indirect addressing on SI, which is implemented using dynamic
 ; indexing of vectors.
@@ -317,7 +317,7 @@ entry:
 
 ; FIXME: Why is vector copied in between?
 
-; GCN-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[IDX0:v[0-9]+]]
 ; GCN-DAG: s_mov_b32 [[S_ELT1:s[0-9]+]], 9
 ; GCN-DAG: s_mov_b32 [[S_ELT0:s[0-9]+]], 7
 ; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], [[S_ELT0]]
@@ -396,7 +396,7 @@ bb2:
 
 ; GCN-LABEL: {{^}}insert_vgpr_offset_multiple_in_block:
 ; GCN-DAG: s_load_dwordx4 s{{\[}}[[S_ELT0:[0-9]+]]:[[S_ELT3:[0-9]+]]{{\]}}
-; GCN-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]]
+; GCN-DAG: {{buffer|flat|global}}_load_dword [[IDX0:v[0-9]+]]
 ; GCN-DAG: v_mov_b32 [[INS0:v[0-9]+]], 62
 
 ; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT3:[0-9]+]], s[[S_ELT3]]
@@ -469,55 +469,6 @@ bb2:
   ret void
 }
 
-; GCN-LABEL: {{^}}extract_adjacent_blocks:
-; GCN: s_load_dword [[ARG:s[0-9]+]]
-; GCN: s_cmp_lg_u32
-; GCN: s_cbranch_scc0 [[BB4:BB[0-9]+_[0-9]+]]
-
-; GCN: buffer_load_dwordx4
-; MOVREL: s_mov_b32 m0,
-; MOVREL: v_movrels_b32_e32
-
-; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, src0
-; IDXMODE: v_mov_b32_e32
-; IDXMODE: s_set_gpr_idx_off
-
-; GCN: s_branch [[ENDBB:BB[0-9]+_[0-9]+]]
-
-; GCN: [[BB4]]:
-; GCN: buffer_load_dwordx4
-; MOVREL: s_mov_b32 m0,
-; MOVREL: v_movrels_b32_e32
-
-; IDXMODE: s_set_gpr_idx_on
-; IDXMODE: v_mov_b32_e32
-; IDXMODE: s_set_gpr_idx_off
-
-; GCN: [[ENDBB]]:
-; GCN: buffer_store_dword
-; GCN: s_endpgm
-define amdgpu_kernel void @extract_adjacent_blocks(i32 %arg) #0 {
-bb:
-  %tmp = icmp eq i32 %arg, 0
-  br i1 %tmp, label %bb1, label %bb4
-
-bb1:
-  %tmp2 = load volatile <4 x float>, <4 x float> addrspace(1)* undef
-  %tmp3 = extractelement <4 x float> %tmp2, i32 undef
-  call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp2) #0 ; Prevent block optimize out
-  br label %bb7
-
-bb4:
-  %tmp5 = load volatile <4 x float>, <4 x float> addrspace(1)* undef
-  %tmp6 = extractelement <4 x float> %tmp5, i32 undef
-  call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp5) #0 ; Prevent block optimize out
-  br label %bb7
-
-bb7:
-  %tmp8 = phi float [ %tmp3, %bb1 ], [ %tmp6, %bb4 ]
-  store volatile float %tmp8, float addrspace(1)* undef
-  ret void
-}
 
 ; GCN-LABEL: {{^}}insert_adjacent_blocks:
 ; GCN: s_load_dword [[ARG:s[0-9]+]]
@@ -603,7 +554,8 @@ bb7:                                              ; preds = %bb4, %bb1
 ; IDXMODE: v_mov_b32_e32 v[[VEC0_ELT2]], -4.0
 ; IDXMODE: s_set_gpr_idx_off
 
-; GCN: s_mov_b32 m0, -1
+; PREGFX9: s_mov_b32 m0, -1
+; GFX9-NOT: s_mov_b32 m0
 ; GCN: ds_write_b32
 ; GCN: ds_write_b32
 ; GCN: s_endpgm
diff --git a/test/CodeGen/AMDGPU/inline-asm.ll b/test/CodeGen/AMDGPU/inline-asm.ll
index 75826d530cb04..2856212bc8993 100644
--- a/test/CodeGen/AMDGPU/inline-asm.ll
+++ b/test/CodeGen/AMDGPU/inline-asm.ll
@@ -246,3 +246,19 @@ entry:
   store i32 %add, i32 addrspace(1)* undef
   ret void
 }
+
+; CHECK-LABEL: {{^}}asm_constraint_c_n:
+; CHECK: s_trap 10{{$}}
+define amdgpu_kernel void @asm_constraint_c_n()  {
+entry:
+  tail call void asm sideeffect "s_trap ${0:c}", "n"(i32 10) #1
+  ret void
+}
+
+; CHECK-LABEL: {{^}}asm_constraint_n_n:
+; CHECK: s_trap -10{{$}}
+define amdgpu_kernel void @asm_constraint_n_n()  {
+entry:
+  tail call void asm sideeffect "s_trap ${0:n}", "n"(i32 10) #1
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/inline-attr.ll b/test/CodeGen/AMDGPU/inline-attr.ll
new file mode 100644
index 0000000000000..6f6b5f4c0b023
--- /dev/null
+++ b/test/CodeGen/AMDGPU/inline-attr.ll
@@ -0,0 +1,33 @@
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-unsafe-fp-math %s  | FileCheck -check-prefix=GCN -check-prefix=UNSAFE %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-no-nans-fp-math %s | FileCheck -check-prefix=GCN -check-prefix=NONANS %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-no-infs-fp-math %s | FileCheck -check-prefix=GCN -check-prefix=NOINFS %s
+
+; GCN: define float @foo(float %x) local_unnamed_addr #0 {
+; GCN: define amdgpu_kernel void @caller(float addrspace(1)* nocapture %p) local_unnamed_addr #1 {
+; GCN: %mul.i = fmul float %load, 1.500000e+01
+
+; UNSAFE: attributes #0 = { norecurse nounwind readnone "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+; UNSAFE: attributes #1 = { norecurse nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+
+; NOINFS: attributes #0 = { norecurse nounwind readnone "no-infs-fp-math"="true" }
+; NOINFS: attributes #1 = { norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="false" "unsafe-fp-math"="false" }
+
+; NONANS: attributes #0 = { norecurse nounwind readnone "no-nans-fp-math"="true" }
+; NONANS: attributes #1 = { norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="true" "unsafe-fp-math"="false" }
+
+define float @foo(float %x) #0 {
+entry:
+  %mul = fmul float %x, 1.500000e+01
+  ret float %mul
+}
+
+define amdgpu_kernel void @caller(float addrspace(1)* %p) #1 {
+entry:
+  %load = load float, float addrspace(1)* %p, align 4
+  %call = call fast float @foo(float %load) #0
+  store float %call, float addrspace(1)* %p, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/AMDGPU/inline-calls.ll b/test/CodeGen/AMDGPU/inline-calls.ll
index f8821f319893c..f686f376837f8 100644
--- a/test/CodeGen/AMDGPU/inline-calls.ll
+++ b/test/CodeGen/AMDGPU/inline-calls.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck  %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck  %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck  %s
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck %s
 
diff --git a/test/CodeGen/AMDGPU/inlineasm-packed.ll b/test/CodeGen/AMDGPU/inlineasm-packed.ll
index 3c6c7e1d1b426..027435421095a 100644
--- a/test/CodeGen/AMDGPU/inlineasm-packed.ll
+++ b/test/CodeGen/AMDGPU/inlineasm-packed.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}inline_asm_input_v2i16:
 ; GCN: s_mov_b32 s{{[0-9]+}}, s{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir b/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
index bd5f296affb54..e3a559998be29 100644
--- a/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
+++ b/test/CodeGen/AMDGPU/insert-skips-kill-uncond.mir
@@ -33,7 +33,7 @@ body: |
   bb.1:
     successors: %bb.2
     %vgpr0 = V_MOV_B32_e32 0, implicit %exec
-    SI_KILL_TERMINATOR %vgpr0, implicit-def %exec, implicit-def %vcc, implicit %exec
+    SI_KILL_F32_COND_IMM_TERMINATOR %vgpr0, 0, 3, implicit-def %exec, implicit-def %vcc, implicit %exec
     S_BRANCH %bb.2
 
   bb.2:
diff --git a/test/CodeGen/AMDGPU/insert_subreg.ll b/test/CodeGen/AMDGPU/insert_subreg.ll
index e895f27c886da..dcf6fe8a3e472 100644
--- a/test/CodeGen/AMDGPU/insert_subreg.ll
+++ b/test/CodeGen/AMDGPU/insert_subreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -mattr=-promote-alloca -verify-machineinstrs < %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=-promote-alloca -verify-machineinstrs < %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s
 
 ; Test that INSERT_SUBREG instructions don't have non-register operands after
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.ll b/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 6391b6b5407b0..79e1943f8fb75 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mattr=+max-private-element-size-16 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=GCN-NO-TONGA %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -mattr=+max-private-element-size-16 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=GCN-TONGA %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tahiti -mattr=+max-private-element-size-16 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=GCN-NO-TONGA %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -mattr=+max-private-element-size-16 < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=GCN-TONGA %s
 
 ; FIXME: Broken on evergreen
 ; FIXME: For some reason the 8 and 16 vectors are being stored as
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index 86fc41a237720..de973eb0ba7e4 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=gfx901 -enable-amdgpu-aa=0 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=fiji -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=VI -check-prefix=GFX89 %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=hawaii -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=CI %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -enable-amdgpu-aa=0 -mattr=+flat-for-global,-fp64-fp16-denormals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=fiji -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=hawaii -enable-amdgpu-aa=0 -mattr=+flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=CIVI -check-prefix=CI %s
 
 ; GCN-LABEL: {{^}}s_insertelement_v2i16_0:
 ; GCN: s_load_dword [[VEC:s[0-9]+]]
@@ -81,8 +81,9 @@ define amdgpu_kernel void @s_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)*
 ; GCN: s_load_dword [[ELT_ARG:s[0-9]+]], s[0:1]
 ; GCN: s_load_dword [[VEC:s[0-9]+]],
 
-; CIVI-DAG: s_and_b32 [[ELT1:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
-; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT0]], [[ELT1]]
+; CIVI-DAG: s_lshr_b32 [[ELT1:s[0-9]+]], [[ELT_ARG]], 16
+; CIVI-DAG: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff0000{{$}}
+; CIVI: s_or_b32 s{{[0-9]+}}, [[ELT1]], [[ELT0]]
 
 ; GFX9: s_lshr_b32 [[ELT1:s[0-9]+]], [[ELT_ARG]], 16
 ; GFX9: s_pack_lh_b32_b16 s{{[0-9]+}}, [[ELT1]], [[VEC]]
@@ -174,7 +175,7 @@ define amdgpu_kernel void @s_insertelement_v2f16_0(<2 x half> addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}s_insertelement_v2f16_1:
-; GFX9: s_load_dword [[VEC:s[0-9]+]]
+; GCN: s_load_dword [[VEC:s[0-9]+]]
 ; GCN-NOT: s_lshr
 
 ; CIVI: s_and_b32 [[ELT0:s[0-9]+]], [[VEC]], 0xffff{{$}}
@@ -189,14 +190,14 @@ define amdgpu_kernel void @s_insertelement_v2f16_1(<2 x half> addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_0:
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 ; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
 ; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 0x3e7, [[ELT1]]
 
 ; GFX9-DAG: s_movk_i32 [[ELT0:s[0-9]+]], 0x3e7{{$}}
 ; GFX9-DAG: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff{{$}}
 ; GFX9: v_bfi_b32 [[RES:v[0-9]+]], [[MASK]], [[ELT0]], [[VEC]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -209,7 +210,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_0_reghi:
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 ; GCN-DAG: s_load_dword [[ELT0:s[0-9]+]]
 
 ; CIVI-DAG: s_lshr_b32 [[ELT0_SHIFT:s[0-9]+]], [[ELT0]], 16
@@ -220,7 +221,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(<2 x i16> addrspace(1)* %out,
 ; GFX9-DAG: v_lshrrev_b32_e64 [[ELT0_SHIFT:v[0-9]+]], 16, [[ELT0]]
 ; GFX9: v_and_or_b32 [[RES:v[0-9]+]], [[VEC]], [[MASK]], [[ELT0_SHIFT]]
 
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, i32 %elt.arg) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -235,7 +236,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_0_inlineimm:
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
 ; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 53, [[ELT1]]
@@ -243,7 +244,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(<2 x i16> addrspace(1)*
 ; GFX9-DAG: v_mov_b32_e32 [[MASK:v[0-9]+]], 0xffff{{$}}
 ; GFX9: v_bfi_b32 [[RES:v[0-9]+]], [[MASK]], 53, [[VEC]]
 
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -259,15 +260,17 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(<2 x i16> addrspace
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_1:
 ; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e70000
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x3e70000, [[VEC]]
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x3e7
 ; GFX9-DAG: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[K]], 16, [[ELT0]]
 
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, [[VEC]]
+; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x3e70000, [[AND]]
+; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_1(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -281,13 +284,13 @@ define amdgpu_kernel void @v_insertelement_v2i16_1(<2 x i16> addrspace(1)* %out,
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_1_inlineimm:
 ; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0xfff10000
-; GCN: flat_load_dword [[VEC:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 ; CI:   v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0xfff10000, [[ELT0]]
 ; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], -15, 16, [[ELT0]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -300,7 +303,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(<2 x i16> addrspace
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_0:
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
 ; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 0x4500, [[ELT1]]
@@ -309,7 +312,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_1_inlineimm(<2 x i16> addrspace
 ; GFX9-DAG: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[VEC]]
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[ELT1]], 16, [[ELT0]]
 
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_0(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -322,14 +325,14 @@ define amdgpu_kernel void @v_insertelement_v2f16_0(<2 x half> addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_0_inlineimm:
-; GCN: flat_load_dword [[VEC:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; CIVI: v_and_b32_e32 [[ELT1:v[0-9]+]], 0xffff0000, [[VEC]]
 ; CIVI: v_or_b32_e32 [[RES:v[0-9]+]], 53, [[ELT1]]
 
 ; GFX9: v_lshrrev_b32_e32 [[ELT1:v[0-9]+]], 16, [[VEC]]
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[ELT1]], 16, 53
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -343,15 +346,18 @@ define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(<2 x half> addrspac
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_1:
 ; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x45000000
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
-; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x45000000, [[VEC]]
-; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x4500
 ; GFX9-DAG: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], [[K]], 16, [[ELT0]]
 
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; CI: v_and_b32_e32 [[AND:v[0-9]+]], 0xffff, [[VEC]]
+; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x45000000, [[AND]]
+
+; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_1(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -365,13 +371,13 @@ define amdgpu_kernel void @v_insertelement_v2f16_1(<2 x half> addrspace(1)* %out
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_1_inlineimm:
 ; VI: v_mov_b32_e32 [[K:v[0-9]+]], 0x230000
-; GCN: flat_load_dword [[VEC:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 ; CI: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VEC]]
 ; CI: v_or_b32_e32 [[RES:v[0-9]+]], 0x230000, [[ELT0]]
 ; VI: v_or_b32_sdwa [[RES:v[0-9]+]], [[VEC]], [[K]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 ; GFX9: v_lshl_or_b32 [[RES:v[0-9]+]], 35, 16, [[ELT0]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RES]]
 define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -392,7 +398,7 @@ define amdgpu_kernel void @v_insertelement_v2f16_1_inlineimm(<2 x half> addrspac
 ; GCN-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 16
 ; GCN-DAG: s_lshl_b32 [[MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
 ; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VVEC]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @s_insertelement_v2i16_dynamic(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(2)* %vec.ptr, i32 addrspace(2)* %idx.ptr) #0 {
   %idx = load volatile i32, i32 addrspace(2)* %idx.ptr
   %vec = load <2 x i16>, <2 x i16> addrspace(2)* %vec.ptr
@@ -402,13 +408,13 @@ define amdgpu_kernel void @s_insertelement_v2i16_dynamic(<2 x i16> addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_dynamic_sgpr:
-; GCN-DAG: flat_load_dword [[VEC:v[0-9]+]]
+; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 ; GCN-DAG: s_load_dword [[IDX:s[0-9]+]]
 ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
 ; GCN-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 16
 ; GCN-DAG: s_lshl_b32 [[MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
 ; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VEC]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, i32 %idx) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -421,11 +427,13 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspac
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_dynamic_vgpr:
-; GCN: flat_load_dword [[IDX:v[0-9]+]]
-; GCN: flat_load_dword [[VEC:v[0-9]+]]
 ; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG:   v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
+; CI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
+
+; GCN: {{flat|global}}_load_dword [[IDX:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
+; GFX89-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7
 ; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 16, [[IDX]]
 ; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]]
 
@@ -433,7 +441,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspac
 ; CI-DAG: v_lshl_b32_e32 [[MASK:v[0-9]+]], 0xffff, [[SCALED_IDX]]
 
 ; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VEC]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_insertelement_v2i16_dynamic_vgpr(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, i32 addrspace(1)* %idx.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -448,11 +456,13 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_vgpr(<2 x i16> addrspac
 }
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_dynamic_vgpr:
-; GCN: flat_load_dword [[IDX:v[0-9]+]]
-; GCN: flat_load_dword [[VEC:v[0-9]+]]
 ; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG:   v_mov_b32_e32 [[K:v[0-9]+]], 0x1234
+; CI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234
+
+; GCN: {{flat|global}}_load_dword [[IDX:v[0-9]+]]
+; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
+; GFX89-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234
 ; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 16, [[IDX]]
 ; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]]
 
@@ -460,7 +470,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_vgpr(<2 x i16> addrspac
 ; CI-DAG: v_lshl_b32_e32 [[MASK:v[0-9]+]], 0xffff, [[SCALED_IDX]]
 
 ; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[MASK]], [[K]], [[VEC]]
-; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
+; GCN: {{flat|global}}_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]]
 define amdgpu_kernel void @v_insertelement_v2f16_dynamic_vgpr(<2 x half> addrspace(1)* %out, <2 x half> addrspace(1)* %in, i32 addrspace(1)* %idx.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/inserted-wait-states.mir b/test/CodeGen/AMDGPU/inserted-wait-states.mir
index c6fe6debd225a..698f2c3ebc477 100644
--- a/test/CodeGen/AMDGPU/inserted-wait-states.mir
+++ b/test/CodeGen/AMDGPU/inserted-wait-states.mir
@@ -13,6 +13,7 @@
   define amdgpu_kernel void @s_mov_fed_b32() { ret void }
   define amdgpu_kernel void @s_movrel() { ret void }
   define amdgpu_kernel void @v_interp() { ret void }
+  define amdgpu_kernel void @dpp() { ret void }
 
   define amdgpu_kernel void @mov_fed_hazard_crash_on_dbg_value(i32 addrspace(1)* %A) {
   entry:
@@ -436,22 +437,22 @@ body: |
 
 # GCN-LABEL: bb.0:
 # GCN: S_MOV_B32
-# GFX9: S_NOP
+# GFX9-NEXT: S_NOP
 # GCN-NEXT: V_INTERP_P1_F32
 
 # GCN-LABEL: bb.1:
 # GCN: S_MOV_B32
-# GFX9: S_NOP
+# GFX9-NEXT: S_NOP
 # GCN-NEXT: V_INTERP_P2_F32
 
 # GCN-LABEL: bb.2:
 # GCN: S_MOV_B32
-# GFX9: S_NOP
+# GFX9-NEXT: S_NOP
 # GCN-NEXT: V_INTERP_P1_F32_16bank
 
 # GCN-LABEL: bb.3:
 # GCN: S_MOV_B32
-# GFX9: S_NOP
+# GFX9-NEXT: S_NOP
 # GCN-NEXT: V_INTERP_MOV_F32
 
 name: v_interp
@@ -477,6 +478,40 @@ body: |
     %vgpr0 = V_INTERP_MOV_F32 0, 0, 0, implicit %m0, implicit %exec
     S_ENDPGM
 ...
+
+...
+---
+
+# GCN-LABEL: name: dpp
+
+# VI-LABEL: bb.0:
+# VI: V_MOV_B32_e32
+# VI-NEXT: S_NOP 0
+# VI-NEXT: S_NOP 0
+# VI-NEXT: V_MOV_B32_dpp
+
+# VI-LABEL: bb.1:
+# VI: V_CMPX_EQ_I32_e32
+# VI-NEXT: S_NOP 0
+# VI-NEXT: S_NOP 0
+# VI-NEXT: S_NOP 0
+# VI-NEXT: S_NOP 0
+# VI-NEXT: S_NOP 0
+# VI-NEXT: V_MOV_B32_dpp
+
+name: dpp
+
+body: |
+  bb.0:
+    %vgpr0 = V_MOV_B32_e32 0, implicit %exec
+    %vgpr1 = V_MOV_B32_dpp %vgpr1, %vgpr0, 0, 15, 15, 0, implicit %exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    implicit %exec, implicit %vcc = V_CMPX_EQ_I32_e32 %vgpr0, %vgpr1, implicit %exec
+    %vgpr3 = V_MOV_B32_dpp %vgpr3, %vgpr0, 0, 15, 15, 0, implicit %exec
+    S_ENDPGM
+...
 ---
 name:            mov_fed_hazard_crash_on_dbg_value
 alignment:       0
@@ -513,7 +548,7 @@ body:             |
 
     %flat_scr_lo = S_ADD_U32 %sgpr6, %sgpr9, implicit-def %scc
     %flat_scr_hi = S_ADDC_U32 %sgpr7, 0, implicit-def %scc, implicit %scc
-    DBG_VALUE _, 2, !5, !11, debug-location !12
+    DBG_VALUE %noreg, 2, !5, !11, debug-location !12
     %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr4_sgpr5, 0, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
     dead %sgpr6_sgpr7 = KILL %sgpr4_sgpr5
     %sgpr8 = S_MOV_B32 %sgpr5
diff --git a/test/CodeGen/AMDGPU/internalize.ll b/test/CodeGen/AMDGPU/internalize.ll
index 968b1d326a76d..8f7e5568b8810 100644
--- a/test/CodeGen/AMDGPU/internalize.ll
+++ b/test/CodeGen/AMDGPU/internalize.ll
@@ -1,35 +1,58 @@
-; RUN: opt -O1 -S -mtriple=amdgcn-unknown-amdhsa -amdgpu-internalize-symbols < %s | FileCheck %s
-; CHECK-NOT: unused
-; CHECK-NOT: foo_used
-; CHECK: gvar_used
-; CHECK: main_kernel
+; RUN: opt -O1 -S -mtriple=amdgcn-unknown-amdhsa -amdgpu-internalize-symbols < %s | FileCheck -check-prefix=ALL -check-prefix=OPT %s
+; RUN: opt -O0 -S -mtriple=amdgcn-unknown-amdhsa -amdgpu-internalize-symbols < %s | FileCheck -check-prefix=ALL -check-prefix=OPTNONE %s
 
+; OPT-NOT: gvar_unused
+; OPTNONE: gvar_unused
 @gvar_unused = addrspace(1) global i32 undef, align 4
+
+; ALL: gvar_used
 @gvar_used = addrspace(1) global i32 undef, align 4
 
-; Function Attrs: alwaysinline nounwind
-define amdgpu_kernel void @foo_unused(i32 addrspace(1)* %out) local_unnamed_addr #1 {
+; ALL: define internal fastcc void @func_used_noinline(
+define fastcc void @func_used_noinline(i32 addrspace(1)* %out, i32 %tid) #1 {
+entry:
+  store volatile i32 %tid, i32 addrspace(1)* %out
+  ret void
+}
+
+; OPTNONE: define internal fastcc void @func_used_alwaysinline(
+; OPT-NOT: @func_used_alwaysinline
+define fastcc void @func_used_alwaysinline(i32 addrspace(1)* %out, i32 %tid) #2 {
 entry:
-  store i32 1, i32 addrspace(1)* %out
+  store volatile i32 %tid, i32 addrspace(1)* %out
   ret void
 }
 
-; Function Attrs: alwaysinline nounwind
-define amdgpu_kernel void @foo_used(i32 addrspace(1)* %out, i32 %tid) local_unnamed_addr #1 {
+; OPTNONE: define internal void @func_unused(
+; OPT-NOT: @func_unused
+define void @func_unused(i32 addrspace(1)* %out, i32 %tid) #1 {
 entry:
-  store i32 %tid, i32 addrspace(1)* %out
+  store volatile i32 %tid, i32 addrspace(1)* %out
   ret void
 }
 
+; ALL: define amdgpu_kernel void @kernel_unused(
+define amdgpu_kernel void @kernel_unused(i32 addrspace(1)* %out) #1 {
+entry:
+  store volatile i32 1, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL: define amdgpu_kernel void @main_kernel()
+; ALL: tail call i32 @llvm.amdgcn.workitem.id.x
+; ALL: tail call fastcc void @func_used_noinline
+; ALL: store volatile
+; ALL: ret void
 define amdgpu_kernel void @main_kernel() {
 entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  tail call void @foo_used(i32 addrspace(1)* @gvar_used, i32 %tid) nounwind
+  %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+  tail call fastcc void @func_used_noinline(i32 addrspace(1)* @gvar_used, i32 %tid)
+  tail call fastcc void @func_used_alwaysinline(i32 addrspace(1)* @gvar_used, i32 %tid)
   ret void
 }
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 attributes #0 = { nounwind readnone }
-
-attributes #1 = { alwaysinline nounwind }
+attributes #1 = { noinline nounwind }
+attributes #2 = { alwaysinline nounwind }
diff --git a/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir b/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
index 67642282f75b0..61aa39fcc25da 100644
--- a/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
+++ b/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
@@ -26,7 +26,7 @@
 ...
 ---
 # CHECK-LABEL: name: invert_br_undef_vcc
-# CHECK: S_CBRANCH_VCCZ %bb.1.else, implicit undef %vcc
+# CHECK: S_CBRANCH_VCCZ %bb.1, implicit undef %vcc
 
 name:            invert_br_undef_vcc
 alignment:       0
@@ -58,7 +58,7 @@ body:             |
     %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
     %sgpr7 = S_MOV_B32 61440
     %sgpr6 = S_MOV_B32 -1
-    S_CBRANCH_VCCNZ %bb.2.if, implicit undef %vcc
+    S_CBRANCH_VCCNZ %bb.2, implicit undef %vcc
 
   bb.1.else:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
@@ -66,7 +66,7 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 100, implicit %exec
     BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     %vgpr0 = V_MOV_B32_e32 1, implicit %exec
-    S_BRANCH %bb.3.done
+    S_BRANCH %bb.3
 
   bb.2.if:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
diff --git a/test/CodeGen/AMDGPU/ipra.ll b/test/CodeGen/AMDGPU/ipra.ll
new file mode 100644
index 0000000000000..276de14612d7e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/ipra.ll
@@ -0,0 +1,109 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -enable-ipra -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -amdgpu-sroa=0 < %s | FileCheck -check-prefix=GCN %s
+
+; Kernels are not called, so there is no call preserved mask.
+; GCN-LABEL: {{^}}kernel:
+; GCN: flat_store_dword
+define amdgpu_kernel void @kernel(i32 addrspace(1)* %out) #0 {
+entry:
+  store i32 0, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}func:
+; GCN: ; NumVgprs: 8
+define void @func() #1 {
+  call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call:
+; GCN-NOT: buffer_store
+; GCN-NOT: buffer_load
+; GCN-NOT: readlane
+; GCN-NOT: writelane
+; GCN: flat_load_dword v8
+; GCN: s_swappc_b64
+; GCN-NOT: buffer_store
+; GCN-NOT: buffer_load
+; GCN-NOT: readlane
+; GCN-NOT: writelane
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v8
+
+; GCN: ; NumSgprs: 37
+; GCN: ; NumVgprs: 9
+define amdgpu_kernel void @kernel_call() #0 {
+  %vgpr = load volatile i32, i32 addrspace(1)* undef
+  tail call void @func()
+  store volatile i32 %vgpr, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_regular_call:
+; GCN-NOT: buffer_store
+; GCN-NOT: buffer_load
+; GCN-NOT: readlane
+; GCN-NOT: writelane
+; GCN: flat_load_dword v8
+; GCN: s_swappc_b64
+; GCN-NOT: buffer_store
+; GCN-NOT: buffer_load
+; GCN-NOT: readlane
+; GCN-NOT: writelane
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v8
+
+; GCN: ; NumSgprs: 32
+; GCN: ; NumVgprs: 9
+define void @func_regular_call() #1 {
+  %vgpr = load volatile i32, i32 addrspace(1)* undef
+  tail call void @func()
+  store volatile i32 %vgpr, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_tail_call:
+; GCN: s_waitcnt
+; GCN-NEXT: s_getpc_b64 s[6:7]
+; GCN-NEXT: s_add_u32 s6,
+; GCN-NEXT: s_addc_u32 s7,
+; GCN-NEXT: s_setpc_b64 s[6:7]
+
+; GCN: ; NumSgprs: 32
+; GCN: ; NumVgprs: 8
+define void @func_tail_call() #1 {
+  tail call void @func()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_tail_call:
+; GCN: flat_load_dword v8
+; GCN: s_swappc_b64
+; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v8
+; GCN: s_setpc_b64
+
+; GCN: ; NumSgprs: 32
+; GCN: ; NumVgprs: 9
+define void @func_call_tail_call() #1 {
+  %vgpr = load volatile i32, i32 addrspace(1)* undef
+  tail call void @func()
+  store volatile i32 %vgpr, i32 addrspace(1)* undef
+  tail call void @func()
+  ret void
+}
+
+define void @void_func_void() noinline {
+  ret void
+}
+
+; Make sure we don't get save/restore of FP between calls.
+; GCN-LABEL: {{^}}test_funcx2:
+; GCN-NOT: s5
+; GCN-NOT: s32
+define void @test_funcx2() #0 {
+  call void @void_func_void()
+  call void @void_func_void()
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/knownbits-recursion.ll b/test/CodeGen/AMDGPU/knownbits-recursion.ll
new file mode 100644
index 0000000000000..9a193031ccdb0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/knownbits-recursion.ll
@@ -0,0 +1,69 @@
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+
+; Check we can compile this test without infinite loop in the
+; DAG.computeKnownBits() due to missing (Depth + 1) argument in
+; call to it from computeKnownBitsForTargetNode().
+
+; Check that we actually have that target 24 bit multiplication
+; node produced.
+
+; GCN: v_mul_u32_u24
+define amdgpu_kernel void @test(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  br label %bb4
+
+bb1:                                              ; preds = %bb4
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i32 %tmp46
+  store i32 %tmp46, i32 addrspace(1)* %tmp3, align 4
+  ret void
+
+bb4:                                              ; preds = %bb4, %bb
+  %tmp5 = phi i32 [ 0, %bb ], [ %tmp87, %bb4 ]
+  %tmp6 = phi i32 [ %tmp, %bb ], [ %tmp46, %bb4 ]
+  %tmp7 = ashr i32 %tmp6, 16
+  %tmp8 = mul nsw i32 %tmp7, %tmp7
+  %tmp9 = lshr i32 %tmp8, 16
+  %tmp10 = mul nuw nsw i32 %tmp9, %tmp9
+  %tmp11 = lshr i32 %tmp10, 16
+  %tmp12 = mul nuw nsw i32 %tmp11, %tmp11
+  %tmp13 = lshr i32 %tmp12, 16
+  %tmp14 = mul nuw nsw i32 %tmp13, %tmp13
+  %tmp15 = lshr i32 %tmp14, 16
+  %tmp16 = mul nuw nsw i32 %tmp15, %tmp15
+  %tmp17 = lshr i32 %tmp16, 16
+  %tmp18 = mul nuw nsw i32 %tmp17, %tmp17
+  %tmp19 = lshr i32 %tmp18, 16
+  %tmp20 = mul nuw nsw i32 %tmp19, %tmp19
+  %tmp21 = lshr i32 %tmp20, 16
+  %tmp22 = mul nuw nsw i32 %tmp21, %tmp21
+  %tmp23 = lshr i32 %tmp22, 16
+  %tmp24 = mul nuw nsw i32 %tmp23, %tmp23
+  %tmp25 = lshr i32 %tmp24, 16
+  %tmp26 = mul nuw nsw i32 %tmp25, %tmp25
+  %tmp27 = lshr i32 %tmp26, 16
+  %tmp28 = mul nuw nsw i32 %tmp27, %tmp27
+  %tmp29 = lshr i32 %tmp28, 16
+  %tmp30 = mul nuw nsw i32 %tmp29, %tmp29
+  %tmp31 = lshr i32 %tmp30, 16
+  %tmp32 = mul nuw nsw i32 %tmp31, %tmp31
+  %tmp33 = lshr i32 %tmp32, 16
+  %tmp34 = mul nuw nsw i32 %tmp33, %tmp33
+  %tmp35 = lshr i32 %tmp34, 16
+  %tmp36 = mul nuw nsw i32 %tmp35, %tmp35
+  %tmp37 = lshr i32 %tmp36, 16
+  %tmp38 = mul nuw nsw i32 %tmp37, %tmp37
+  %tmp39 = lshr i32 %tmp38, 16
+  %tmp40 = mul nuw nsw i32 %tmp39, %tmp39
+  %tmp41 = lshr i32 %tmp40, 16
+  %tmp42 = mul nuw nsw i32 %tmp41, %tmp41
+  %tmp43 = lshr i32 %tmp42, 16
+  %tmp44 = mul nuw nsw i32 %tmp43, %tmp43
+  %tmp45 = lshr i32 %tmp44, 16
+  %tmp46 = mul nuw nsw i32 %tmp45, %tmp45
+  %tmp87 = add nuw nsw i32 %tmp5, 1
+  %tmp88 = icmp eq i32 %tmp87, 1000
+  br i1 %tmp88, label %bb1, label %bb4
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/test/CodeGen/AMDGPU/large-constant-initializer.ll b/test/CodeGen/AMDGPU/large-constant-initializer.ll
index c46d68e38ade8..b762a91cea5fc 100644
--- a/test/CodeGen/AMDGPU/large-constant-initializer.ll
+++ b/test/CodeGen/AMDGPU/large-constant-initializer.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI < %s
+; RUN: llc -march=amdgcn -mcpu=tahiti < %s
 ; RUN: llc -march=amdgcn -mcpu=tonga < %s
 ; CHECK: s_endpgm
 
diff --git a/test/CodeGen/AMDGPU/lds-initializer.ll b/test/CodeGen/AMDGPU/lds-initializer.ll
index 254673d8a1e4c..5ced7100596e1 100644
--- a/test/CodeGen/AMDGPU/lds-initializer.ll
+++ b/test/CodeGen/AMDGPU/lds-initializer.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -march=amdgcn -mcpu=SI < %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mcpu=tahiti < %s 2>&1 | FileCheck %s
 ; RUN: not llc -march=amdgcn -mcpu=tonga < %s 2>&1 | FileCheck %s
 
 ; CHECK: in function load_init_lds_global{{.*}}: unsupported initializer for address space
diff --git a/test/CodeGen/AMDGPU/lds-output-queue.ll b/test/CodeGen/AMDGPU/lds-output-queue.ll
index 8b7e9e6d6aa8b..f8fb12eefa620 100644
--- a/test/CodeGen/AMDGPU/lds-output-queue.ll
+++ b/test/CodeGen/AMDGPU/lds-output-queue.ll
@@ -45,21 +45,21 @@ declare void @llvm.r600.group.barrier() nounwind convergent
 ;  %2 = load i32, i32 addrspace(1)* %in
 ;
 ; The instruction selection phase will generate ISA that looks like this:
-; %OQAP = LDS_READ_RET
-; %vreg0 = MOV %OQAP
-; %vreg1 = VTX_READ_32
-; %vreg2 = ADD_INT %vreg1, %vreg0
+; %oqap = LDS_READ_RET
+; %0 = MOV %oqap
+; %1 = VTX_READ_32
+; %2 = ADD_INT %1, %0
 ;
 ; The bottom scheduler will schedule the two ALU instructions first:
 ;
 ; UNSCHEDULED:
-; %OQAP = LDS_READ_RET
-; %vreg1 = VTX_READ_32
+; %oqap = LDS_READ_RET
+; %1 = VTX_READ_32
 ;
 ; SCHEDULED:
 ;
-; vreg0 = MOV %OQAP
-; vreg2 = ADD_INT %vreg1, %vreg2
+; %0 = MOV %oqap
+; %2 = ADD_INT %1, %2
 ;
 ; The lack of proper aliasing results in the local memory read (LDS_READ_RET)
 ; to consider the global memory read (VTX_READ_32) has a chain dependency, so
@@ -67,14 +67,14 @@ declare void @llvm.r600.group.barrier() nounwind convergent
 ; final program which looks like this:
 ;
 ; Alu clause:
-; %OQAP = LDS_READ_RET
+; %oqap = LDS_READ_RET
 ; VTX clause:
-; %vreg1 = VTX_READ_32
+; %1 = VTX_READ_32
 ; Alu clause:
-; vreg0 = MOV %OQAP
-; vreg2 = ADD_INT %vreg1, %vreg2
+; %0 = MOV %oqap
+; %2 = ADD_INT %1, %2
 ;
-; This is an illegal program because the OQAP def and use know occur in
+; This is an illegal program because the oqap def and use know occur in
 ; different ALU clauses.
 ;
 ; This test checks this scenario and makes sure it doesn't result in an
diff --git a/test/CodeGen/AMDGPU/lds-zero-initializer.ll b/test/CodeGen/AMDGPU/lds-zero-initializer.ll
index 53c1c727a19d2..a3b3d7b341fac 100644
--- a/test/CodeGen/AMDGPU/lds-zero-initializer.ll
+++ b/test/CodeGen/AMDGPU/lds-zero-initializer.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -march=amdgcn -mcpu=SI < %s 2>&1 | FileCheck %s
+; RUN: not llc -march=amdgcn -mcpu=tahiti < %s 2>&1 | FileCheck %s
 ; RUN: not llc -march=amdgcn -mcpu=tonga < %s 2>&1 | FileCheck %s
 
 ; CHECK: in function load_zeroinit_lds_global{{.*}}: unsupported initializer for address space
diff --git a/test/CodeGen/AMDGPU/liveness.mir b/test/CodeGen/AMDGPU/liveness.mir
index 6fd8466492d08..8bb946da9ad27 100644
--- a/test/CodeGen/AMDGPU/liveness.mir
+++ b/test/CodeGen/AMDGPU/liveness.mir
@@ -6,7 +6,7 @@
 # liveranges needed it.
 #
 # Should see three distinct value numbers:
-# CHECK: %vreg0 [{{.*}}:0)[{{.*}}:1)[{{.*}}:2) 0@{{[0-9]+[Berd]}} 1@{{[0-9]+[Berd]}} 2@{{[0-9]+B-phi}}
+# CHECK: %0 [{{.*}}:0)[{{.*}}:1)[{{.*}}:2) 0@{{[0-9]+[Berd]}} 1@{{[0-9]+[Berd]}} 2@{{[0-9]+B-phi}}
 --- |
   define amdgpu_kernel void @test0() { ret void }
 ...
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
index 1fc77893e7e97..80a08acfc98e3 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
 declare i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* nocapture, i32, i32, i32, i1) #2
 declare i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
@@ -13,6 +14,8 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
 
 ; Make sure no crash on invalid non-constant
 ; GCN-LABEL: {{^}}invalid_variable_order_lds_atomic_dec_ret_i32:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @invalid_variable_order_lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr, i32 %order.var) #0 {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 %order.var, i32 0, i1 false)
   store i32 %result, i32 addrspace(1)* %out
@@ -21,6 +24,8 @@ define amdgpu_kernel void @invalid_variable_order_lds_atomic_dec_ret_i32(i32 add
 
 ; Make sure no crash on invalid non-constant
 ; GCN-LABEL: {{^}}invalid_variable_scope_lds_atomic_dec_ret_i32:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @invalid_variable_scope_lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr, i32 %scope.var) #0 {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 %scope.var, i1 false)
   store i32 %result, i32 addrspace(1)* %out
@@ -36,7 +41,10 @@ define amdgpu_kernel void @invalid_variable_volatile_lds_atomic_dec_ret_i32(i32
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]]
 define amdgpu_kernel void @lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
@@ -45,7 +53,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_dec_ret_i32_offset:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_dec_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]] offset:16
 define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
@@ -54,18 +65,24 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i32:
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_dec_u32 [[VPTR]], [[DATA]]
 define amdgpu_kernel void @lds_atomic_dec_noret_i32(i32 addrspace(3)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i32_offset:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; GCN-LABEL: {{^}}lds_atomic_dec_noret_i32_offset:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_dec_u32 v{{[0-9]+}}, [[K]] offset:16
 define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
@@ -75,7 +92,8 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(i32 addrspace(3)* %pt
 
 ; GCN-LABEL: {{^}}global_atomic_dec_ret_i32:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; GFX9: global_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]], off glc{{$}}
 define amdgpu_kernel void @global_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32 addrspace(1)* %out
@@ -84,7 +102,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(i32 addrspace(1)* %out, i32
 
 ; GCN-LABEL: {{^}}global_atomic_dec_ret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}}
+; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}}
+; GFX9: global_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]], off offset:16 glc{{$}}
 define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -92,16 +111,19 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(i32 addrspace(1)* %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_dec_noret_i32:
-; GCN: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GCN-LABEL: {{^}}global_atomic_dec_noret_i32:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GFX9: global_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]], off{{$}}
 define amdgpu_kernel void @global_atomic_dec_noret_i32(i32 addrspace(1)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_dec_noret_i32_offset:
+; GCN-LABEL: {{^}}global_atomic_dec_noret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}}
+; CIVI: buffer_atomic_dec [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}}
+; GFX9: global_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]], off offset:16{{$}}
 define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(i32 addrspace(1)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -145,7 +167,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(i32 addrspace(4)* %out, i32 a
 
 ; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16 glc{{$}}
 define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -153,16 +176,18 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(i32 addrspace(4)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i32:
+; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
 define amdgpu_kernel void @flat_atomic_dec_noret_i32(i32 addrspace(4)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i32_offset:
+; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16{{$}}
 define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(i32 addrspace(4)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.dec.i32.p4i32(i32 addrspace(4)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -171,7 +196,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(i32 addrspace(4)* %p
 
 ; GCN-LABEL: {{^}}flat_atomic_dec_ret_i32_offset_addr64:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; CIVI: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; GFX9: flat_atomic_dec v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20 glc{{$}}
 define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id
@@ -184,7 +210,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(i32 addrspace(4
 
 ; GCN-LABEL: {{^}}flat_atomic_dec_noret_i32_offset_addr64:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; CIVI: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; GFX9: flat_atomic_dec v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20{{$}}
 define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(i32 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id
@@ -206,7 +233,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(i64 addrspace(4)* %out, i64 a
 ; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32 glc{{$}}
 define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 {
   %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -214,7 +242,7 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(i64 addrspace(4)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i64:
+; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
@@ -223,10 +251,11 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(i64 addrspace(4)* %ptr) nou
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_dec_noret_i64_offset:
+; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}}
 define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(i64 addrspace(4)* %ptr) nounwind {
   %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p4i64(i64 addrspace(4)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -236,7 +265,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(i64 addrspace(4)* %p
 ; GCN-LABEL: {{^}}flat_atomic_dec_ret_i64_offset_addr64:
 ; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40 glc{{$}}
 define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id
@@ -250,7 +280,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(i64 addrspace(4
 ; GCN-LABEL: {{^}}flat_atomic_dec_noret_i64_offset_addr64:
 ; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; CIVI: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; GFX9: flat_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40{{$}}
 define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(i64 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id
@@ -261,9 +292,12 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(i64 addrspace
 
 @lds0 = addrspace(3) global [512 x i32] undef
 
-; SI-LABEL: {{^}}atomic_dec_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_dec_rtn_u32 {{v[0-9]+}}, [[PTR]] offset:8
+; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_dec_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -275,6 +309,9 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}}
@@ -285,6 +322,9 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_dec_ret_i64_offset:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32
@@ -295,7 +335,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i64:
+; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_dec_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}}
@@ -304,7 +347,10 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i64(i64 addrspace(3)* %ptr) noun
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_dec_noret_i64_offset:
+; GCN-LABEL: {{^}}lds_atomic_dec_noret_i64_offset:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_dec_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}}
@@ -317,7 +363,8 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(i64 addrspace(3)* %pt
 ; GCN-LABEL: {{^}}global_atomic_dec_ret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off glc{{$}}
 define amdgpu_kernel void @global_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64 addrspace(1)* %out
@@ -327,7 +374,8 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(i64 addrspace(1)* %out, i64
 ; GCN-LABEL: {{^}}global_atomic_dec_ret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}}
+; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}}
+; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -335,19 +383,21 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(i64 addrspace(1)* %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_dec_noret_i64:
+; GCN-LABEL: {{^}}global_atomic_dec_noret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off{{$}}
 define amdgpu_kernel void @global_atomic_dec_noret_i64(i64 addrspace(1)* %ptr) nounwind {
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_dec_noret_i64_offset:
+; GCN-LABEL: {{^}}global_atomic_dec_noret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}}
+; CIVI: buffer_atomic_dec_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}}
+; GFX9: global_atomic_dec_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off offset:32{{$}}
 define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(i64 addrspace(1)* %ptr) nounwind {
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.dec.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -387,7 +437,10 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(i64 addrspa
 @lds1 = addrspace(3) global [512 x i64] undef, align 8
 
 ; GCN-LABEL: {{^}}atomic_dec_shl_base_lds_0_i64:
-; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 3, {{v[0-9]+}}
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 3, {{v[0-9]+}}
 ; GCN: ds_dec_rtn_u64 v{{\[[0-9]+:[0-9]+\]}}, [[PTR]], v{{\[[0-9]+:[0-9]+\]}} offset:16
 define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
@@ -402,12 +455,3 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out,
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind argmemonly }
-
-
-
-
-
-
-
-
-
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
index eee8351de79be..75ce7f54ae39c 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 declare i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* nocapture, i32, i32, i32, i1) #2
 declare i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* nocapture, i32, i32, i32, i1) #2
@@ -12,7 +13,10 @@ declare i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* nocapture, i64,
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
 ; GCN-LABEL: {{^}}lds_atomic_inc_ret_i32:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_inc_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]]
 define amdgpu_kernel void @lds_atomic_inc_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 {
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
@@ -21,7 +25,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_inc_ret_i32_offset:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_inc_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]] offset:16
 define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
@@ -30,18 +37,24 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(i32 addrspace(1)* %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i32:
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-LABEL: {{^}}lds_atomic_inc_noret_i32:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_inc_u32 [[VPTR]], [[DATA]]
 define amdgpu_kernel void @lds_atomic_inc_noret_i32(i32 addrspace(3)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i32_offset:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; GCN-LABEL: {{^}}lds_atomic_inc_noret_i32_offset:
+; CIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: ds_inc_u32 v{{[0-9]+}}, [[K]] offset:16
 define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(3)* %ptr, i32 4
@@ -51,7 +64,8 @@ define amdgpu_kernel void @lds_atomic_inc_noret_i32_offset(i32 addrspace(3)* %pt
 
 ; GCN-LABEL: {{^}}global_atomic_inc_ret_i32:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; CIVI: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; GFX9: global_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]], off glc{{$}}
 define amdgpu_kernel void @global_atomic_inc_ret_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, i32 addrspace(1)* %out
@@ -60,7 +74,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(i32 addrspace(1)* %out, i32
 
 ; GCN-LABEL: {{^}}global_atomic_inc_ret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}}
+; CIVI: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 glc{{$}}
+; GFX9: global_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]], off offset:16 glc{{$}}
 define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -68,16 +83,19 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(i32 addrspace(1)* %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_inc_noret_i32:
-; GCN: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GCN-LABEL: {{^}}global_atomic_inc_noret_i32:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; CIVI: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; GFX9: global_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]], off{{$}}
 define amdgpu_kernel void @global_atomic_inc_noret_i32(i32 addrspace(1)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_inc_noret_i32_offset:
+; GCN-LABEL: {{^}}global_atomic_inc_noret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}}
+; CIVI: buffer_atomic_inc [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16{{$}}
+; GFX9: global_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]], off offset:16{{$}}
 define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(i32 addrspace(1)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p1i32(i32 addrspace(1)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -146,7 +164,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(i64 addrspace(1)* %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i64:
+; GCN-LABEL: {{^}}lds_atomic_inc_noret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_inc_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}{{$}}
@@ -155,7 +173,7 @@ define amdgpu_kernel void @lds_atomic_inc_noret_i64(i64 addrspace(3)* %ptr) noun
   ret void
 }
 
-; FUNC-LABEL: {{^}}lds_atomic_inc_noret_i64_offset:
+; GCN-LABEL: {{^}}lds_atomic_inc_noret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: ds_inc_u64 v{{[0-9]+}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}}
@@ -168,7 +186,8 @@ define amdgpu_kernel void @lds_atomic_inc_noret_i64_offset(i64 addrspace(3)* %pt
 ; GCN-LABEL: {{^}}global_atomic_inc_ret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; CIVI: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 glc{{$}}
+; GFX9: global_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off glc{{$}}
 define amdgpu_kernel void @global_atomic_inc_ret_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, i64 addrspace(1)* %out
@@ -178,7 +197,8 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(i64 addrspace(1)* %out, i64
 ; GCN-LABEL: {{^}}global_atomic_inc_ret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}}
+; CIVI: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32 glc{{$}}
+; GFX9: global_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off offset:32 glc{{$}}
 define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(1)* %ptr) #0 {
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -186,19 +206,22 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(i64 addrspace(1)* %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_inc_noret_i64:
+; GCN-LABEL: {{^}}global_atomic_inc_noret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; CIVI: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+
+; GFX9: global_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off{{$}}
 define amdgpu_kernel void @global_atomic_inc_noret_i64(i64 addrspace(1)* %ptr) nounwind {
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %ptr, i64 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}global_atomic_inc_noret_i64_offset:
+; GCN-LABEL: {{^}}global_atomic_inc_noret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}}
+; CIVI: buffer_atomic_inc_x2 v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:32{{$}}
+; GFX9: global_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}}, off offset:32{{$}}
 define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(i64 addrspace(1)* %ptr) nounwind {
   %gep = getelementptr i64, i64 addrspace(1)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p1i64(i64 addrspace(1)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -246,7 +269,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(i32 addrspace(4)* %out, i32 a
 
 ; GCN-LABEL: {{^}}flat_atomic_inc_ret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; CIVI: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; GFX9: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16 glc{{$}}
 define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 {
   %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -254,16 +278,18 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(i32 addrspace(4)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i32:
+; GCN-LABEL: {{^}}flat_atomic_inc_noret_i32:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
 ; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
 define amdgpu_kernel void @flat_atomic_inc_noret_i32(i32 addrspace(4)* %ptr) nounwind {
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i32_offset:
+; GCN-LABEL: {{^}}flat_atomic_inc_noret_i32_offset:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; CIVI: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; GFX9: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:16{{$}}
 define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(i32 addrspace(4)* %ptr) nounwind {
   %gep = getelementptr i32, i32 addrspace(4)* %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p4i32(i32 addrspace(4)* %gep, i32 42, i32 0, i32 0, i1 false)
@@ -272,7 +298,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(i32 addrspace(4)* %p
 
 ; GCN-LABEL: {{^}}flat_atomic_inc_ret_i32_offset_addr64:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; CIVI: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] glc{{$}}
+; GFX9: flat_atomic_inc v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20 glc{{$}}
 define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(i32 addrspace(4)* %out, i32 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id
@@ -285,7 +312,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(i32 addrspace(4
 
 ; GCN-LABEL: {{^}}flat_atomic_inc_noret_i32_offset_addr64:
 ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
-; GCN: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; CIVI: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]]{{$}}
+; GFX9: flat_atomic_inc v{{\[[0-9]+:[0-9]+\]}}, [[K]] offset:20{{$}}
 define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(i32 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, i32 addrspace(4)* %ptr, i32 %id
@@ -322,7 +350,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(i64 addrspace(4)* %out, i64 a
 ; GCN-LABEL: {{^}}flat_atomic_inc_ret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; CIVI: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; GFX9: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32 glc{{$}}
 define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 {
   %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -330,7 +359,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(i64 addrspace(4)* %out
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i64:
+; GCN-LABEL: {{^}}flat_atomic_inc_noret_i64:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
 ; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
@@ -339,10 +368,11 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(i64 addrspace(4)* %ptr) nou
   ret void
 }
 
-; FUNC-LABEL: {{^}}flat_atomic_inc_noret_i64_offset:
+; GCN-LABEL: {{^}}flat_atomic_inc_noret_i64_offset:
 ; GCN-DAG: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN-DAG: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; CIVI: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; GFX9: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:32{{$}}
 define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(i64 addrspace(4)* %ptr) nounwind {
   %gep = getelementptr i64, i64 addrspace(4)* %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p4i64(i64 addrspace(4)* %gep, i64 42, i32 0, i32 0, i1 false)
@@ -352,7 +382,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(i64 addrspace(4)* %p
 ; GCN-LABEL: {{^}}flat_atomic_inc_ret_i64_offset_addr64:
 ; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; CIVI: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} glc{{$}}
+; GFX9: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40 glc{{$}}
 define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(i64 addrspace(4)* %out, i64 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id
@@ -366,7 +397,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(i64 addrspace(4
 ; GCN-LABEL: {{^}}flat_atomic_inc_noret_i64_offset_addr64:
 ; GCN: v_mov_b32_e32 v[[KLO:[0-9]+]], 42
 ; GCN: v_mov_b32_e32 v[[KHI:[0-9]+]], 0{{$}}
-; GCN: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; CIVI: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]$}}
+; GFX9: flat_atomic_inc_x2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[KLO]]:[[KHI]]{{\]}} offset:40{{$}}
 define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(i64 addrspace(4)* %ptr) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, i64 addrspace(4)* %ptr, i32 %id
@@ -375,6 +407,19 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(i64 addrspace
   ret void
 }
 
+; GCN-LABEL: {{^}}nocse_lds_atomic_inc_ret_i32:
+; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 42
+; GCN: ds_inc_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]]
+; GCN: ds_inc_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[K]]
+define amdgpu_kernel void @nocse_lds_atomic_inc_ret_i32(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 addrspace(3)* %ptr) #0 {
+  %result0 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
+  %result1 = call i32 @llvm.amdgcn.atomic.inc.i32.p3i32(i32 addrspace(3)* %ptr, i32 42, i32 0, i32 0, i1 false)
+
+  store i32 %result0, i32 addrspace(1)* %out0
+  store i32 %result1, i32 addrspace(1)* %out1
+  ret void
+}
+
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind argmemonly }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll
index 98f7058b5ef80..0ce7d3efe4545 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.atomic.ll
@@ -2,8 +2,9 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s -check-prefix=CHECK -check-prefix=VI
 
 ;CHECK-LABEL: {{^}}test1:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_atomic_swap v0, off, s[0:3], 0 glc
-;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1fff
+;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1ffc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_swap v0, v1, s[0:3], 0 idxen glc
 ;CHECK: s_waitcnt vmcnt(0)
@@ -14,7 +15,7 @@
 ;CHECK: buffer_atomic_swap v0, v2, s[0:3], 0 offen offset:42 glc
 ;CHECK-DAG: s_waitcnt vmcnt(0)
 ;SICI: buffer_atomic_swap v0, v1, s[0:3], 0 offen glc
-;VI: buffer_atomic_swap v0, off, s[0:3], [[SOFS]] offset:1 glc
+;VI: buffer_atomic_swap v0, off, s[0:3], [[SOFS]] offset:4 glc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_swap v0, off, s[0:3], 0{{$}}
 define amdgpu_ps float @test1(<4 x i32> inreg %rsrc, i32 %data, i32 %vindex, i32 %voffset) {
@@ -32,6 +33,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}test2:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_atomic_add v0, v1, s[0:3], 0 idxen glc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_sub v0, v1, s[0:3], 0 idxen glc
@@ -69,26 +71,27 @@ main_body:
 ; create copies which we don't bother to track here.
 ;
 ;CHECK-LABEL: {{^}}test3:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 0 glc
 ;CHECK: s_waitcnt vmcnt(0)
-;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1fff
+;VI: s_movk_i32 [[SOFS:s[0-9]+]], 0x1ffc
 ;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v2, s[0:3], 0 idxen glc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v3, s[0:3], 0 offen glc
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v[2:3], s[0:3], 0 idxen offen glc
 ;CHECK: s_waitcnt vmcnt(0)
-;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v3, s[0:3], 0 offen offset:42 glc
+;CHECK: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, v3, s[0:3], 0 offen offset:44 glc
 ;CHECK-DAG: s_waitcnt vmcnt(0)
 ;SICI: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen glc
-;VI: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[SOFS]] offset:1 glc
+;VI: buffer_atomic_cmpswap {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[SOFS]] offset:4 glc
 define amdgpu_ps float @test3(<4 x i32> inreg %rsrc, i32 %data, i32 %cmp, i32 %vindex, i32 %voffset) {
 main_body:
   %o1 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %data, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 0, i1 0)
   %o2 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o1, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 0, i1 0)
   %o3 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o2, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 %voffset, i1 0)
   %o4 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o3, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i1 0)
-  %ofs.5 = add i32 %voffset, 42
+  %ofs.5 = add i32 %voffset, 44
   %o5 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o4, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 %ofs.5, i1 0)
   %o6 = call i32 @llvm.amdgcn.buffer.atomic.cmpswap(i32 %o5, i32 %cmp, <4 x i32> %rsrc, i32 0, i32 8192, i1 0)
 
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll
index 9cb9f25520b8b..d5159934d3f92 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll
@@ -27,20 +27,20 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_immoffs_large:
-;SICI: v_mov_b32_e32 [[VOFS:v[0-9]+]], 0x103c
+;SICI: v_mov_b32_e32 [[VOFS:v[0-9]+]], 0x1038
 ;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, [[VOFS]], s[0:3], 0 offen
 ;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 61 offset:4095
-;VI-DAG: s_movk_i32 [[OFS1:s[0-9]+]], 0x7fff
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS1]] offset:4093
+;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 60 offset:4092
+;VI-DAG: s_movk_i32 [[OFS1:s[0-9]+]], 0x7ffc
+;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS1]] offset:4092
 ;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen
-;VI-DAG: s_mov_b32 [[OFS2:s[0-9]+]], 0x8fff
-;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS2]] offset:1
+;VI-DAG: s_mov_b32 [[OFS2:s[0-9]+]], 0x8ffc
+;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS2]] offset:4
 ;CHECK: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
 main_body:
-  %d.0 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 4156, i1 0, i1 0)
-  %d.1 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 36860, i1 0, i1 0)
+  %d.0 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 4152, i1 0, i1 0)
+  %d.1 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 36856, i1 0, i1 0)
   %d.2 = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 36864, i1 0, i1 0)
   %d.3 = fadd <4 x float> %d.0, %d.1
   %data = fadd <4 x float> %d.2, %d.3
@@ -48,10 +48,10 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_immoffs_reuse:
-;VI: s_movk_i32 [[OFS:s[0-9]+]], 0xfff
-;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:65
+;VI: s_movk_i32 [[OFS:s[0-9]+]], 0xffc
+;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:68
 ;VI-NOT: s_mov
-;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:81
+;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS]] offset:84
 ;VI: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_immoffs_reuse(<4 x i32> inreg) {
 main_body:
@@ -80,11 +80,11 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_ofs_imm:
-;CHECK: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 offen offset:58
+;CHECK: buffer_load_format_xyzw v[0:3], v0, s[0:3], 0 offen offset:60
 ;CHECK: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) {
 main_body:
-  %ofs = add i32 %1, 58
+  %ofs = add i32 %1, 60
   %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %0, i32 0, i32 %ofs, i1 0, i1 0)
   ret <4 x float> %data
 }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
index 5c93ae0e78672..49ca7d405724b 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
@@ -18,18 +18,18 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_immoffs:
-;CHECK: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:42
+;CHECK: buffer_load_dwordx4 v[0:3], off, s[0:3], 0 offset:40
 ;CHECK: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_immoffs(<4 x i32> inreg) {
 main_body:
-  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 42, i1 0, i1 0)
+  %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 40, i1 0, i1 0)
   ret <4 x float> %data
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_immoffs_large:
 ;SICI: buffer_load_dwordx4 v[0:3], {{v[0-9]+}}, s[0:3], 0 offen
-;VI: s_movk_i32 [[OFFSET:s[0-9]+]], 0x1fff
-;VI: buffer_load_dwordx4 v[0:3], off, s[0:3], [[OFFSET]] offset:1
+;VI: s_movk_i32 [[OFFSET:s[0-9]+]], 0x1ffc
+;VI: buffer_load_dwordx4 v[0:3], off, s[0:3], [[OFFSET]] offset:4
 ;CHECK: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
 main_body:
@@ -56,11 +56,11 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_ofs_imm:
-;CHECK: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:58
+;CHECK: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen offset:60
 ;CHECK: s_waitcnt
 define amdgpu_ps <4 x float> @buffer_load_ofs_imm(<4 x i32> inreg, i32) {
 main_body:
-  %ofs = add i32 %1, 58
+  %ofs = add i32 %1, 60
   %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %0, i32 0, i32 %ofs, i1 0, i1 0)
   ret <4 x float> %data
 }
@@ -103,7 +103,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_load_negative_offset:
-;CHECK: v_add_i32_e32 [[VOFS:v[0-9]+]], vcc, -16, v0
+;CHECK: v_add_{{[iu]}}32_e32 [[VOFS:v[0-9]+]], vcc, -16, v0
 ;CHECK: buffer_load_dwordx4 v[0:3], [[VOFS]], s[0:3], 0 offen
 define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) {
 main_body:
@@ -126,8 +126,110 @@ entry:
   ret float %val
 }
 
+;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
+main_body:
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a3, i1 0, i1 0)
+  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a4, i1 0, i1 0)
+  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a5, i1 0, i1 0)
+  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a6, i1 0, i1 0)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged_glc_slc:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:12 glc{{$}}
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 glc slc{{$}}
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x1_offen_merged_glc_slc(<4 x i32> inreg %rsrc, i32 %a) {
+main_body:
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a3, i1 1, i1 0)
+  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a4, i1 1, i1 0)
+  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a5, i1 1, i1 1)
+  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %a6, i1 1, i1 1)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_load_x2_offen_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
+main_body:
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 12
+  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  %vr2 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  %r1 = extractelement <2 x float> %vr1, i32 0
+  %r2 = extractelement <2 x float> %vr1, i32 1
+  %r3 = extractelement <2 x float> %vr2, i32 0
+  %r4 = extractelement <2 x float> %vr2, i32 1
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_load_x1_offset_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x1_offset_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %r1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
+  %r2 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+  %r3 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
+  %r4 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
+  %r5 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 28, i1 0, i1 0)
+  %r6 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 32, i1 0, i1 0)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_load_x2_offset_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x2_offset_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %vr1 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
+  %vr2 = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
+  %r1 = extractelement <2 x float> %vr1, i32 0
+  %r2 = extractelement <2 x float> %vr1, i32 1
+  %r3 = extractelement <2 x float> %vr2, i32 0
+  %r4 = extractelement <2 x float> %vr2, i32 1
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  ret void
+}
+
 declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #0
 declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #0
 declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 
 attributes #0 = { nounwind readonly }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
index e50455f6f9a13..7fc7acdff92c1 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.format.ll
@@ -2,6 +2,7 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
 
 ;CHECK-LABEL: {{^}}buffer_store:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], off, s[0:3], 0
 ;CHECK: buffer_store_format_xyzw v[4:7], off, s[0:3], 0 glc
 ;CHECK: buffer_store_format_xyzw v[8:11], off, s[0:3], 0 slc
@@ -14,6 +15,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_immoffs:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], off, s[0:3], 0 offset:42
 define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 main_body:
@@ -22,6 +24,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_idx:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
 main_body:
@@ -30,6 +33,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_ofs:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 offen
 define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
 main_body:
@@ -38,6 +42,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_both:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], v[4:5], s[0:3], 0 idxen offen
 define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32) {
 main_body:
@@ -47,6 +52,7 @@ main_body:
 
 ;CHECK-LABEL: {{^}}buffer_store_both_reversed:
 ;CHECK: v_mov_b32_e32 v6, v4
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], v[5:6], s[0:3], 0 idxen offen
 define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>, i32, i32) {
 main_body:
@@ -57,6 +63,7 @@ main_body:
 ; Ideally, the register allocator would avoid the wait here
 ;
 ;CHECK-LABEL: {{^}}buffer_store_wait:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
 ;CHECK: s_waitcnt expcnt(0)
 ;CHECK: buffer_load_format_xyzw v[0:3], v5, s[0:3], 0 idxen
@@ -71,6 +78,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_x1:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_x v0, v1, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %index) {
 main_body:
@@ -79,6 +87,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_x2:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_format_xy v[0:1], v2, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %index) {
 main_body:
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll
index 81597516d5f2a..c12c713cdc208 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.store.ll
@@ -2,6 +2,7 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
 
 ;CHECK-LABEL: {{^}}buffer_store:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ;CHECK: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 glc
 ;CHECK: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 slc
@@ -14,6 +15,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_immoffs:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:42
 define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 main_body:
@@ -22,6 +24,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_idx:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
 main_body:
@@ -30,6 +33,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_ofs:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 offen
 define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
 main_body:
@@ -38,6 +42,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_both:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], v[4:5], s[0:3], 0 idxen offen
 define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32) {
 main_body:
@@ -47,6 +52,7 @@ main_body:
 
 ;CHECK-LABEL: {{^}}buffer_store_both_reversed:
 ;CHECK: v_mov_b32_e32 v6, v4
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], v[5:6], s[0:3], 0 idxen offen
 define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>, i32, i32) {
 main_body:
@@ -57,6 +63,7 @@ main_body:
 ; Ideally, the register allocator would avoid the wait here
 ;
 ;CHECK-LABEL: {{^}}buffer_store_wait:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[0:3], v4, s[0:3], 0 idxen
 ;CHECK: s_waitcnt expcnt(0)
 ;CHECK: buffer_load_dwordx4 v[0:3], v5, s[0:3], 0 idxen
@@ -71,6 +78,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_x1:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dword v0, v1, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %index) {
 main_body:
@@ -79,6 +87,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}buffer_store_x2:
+;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 idxen
 define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data, i32 %index) #0 {
 main_body:
@@ -86,6 +95,81 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
+define amdgpu_ps void @buffer_store_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 %a3, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 %a4, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 %a5, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 %a6, i1 0, i1 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged_glc_slc:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:12 glc{{$}}
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28 glc slc{{$}}
+define amdgpu_ps void @buffer_store_x1_offen_merged_glc_slc(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 %a3, i1 1, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 %a4, i1 1, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 %a5, i1 1, i1 1)
+  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 %a6, i1 1, i1 1)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_x2_offen_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+define amdgpu_ps void @buffer_store_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a, <2 x float> %v1, <2 x float> %v2) {
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 12
+  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 %a1, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 %a2, i1 0, i1 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_x1_offset_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+define amdgpu_ps void @buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  call void @llvm.amdgcn.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 0, i32 8, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 0, i32 16, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 0, i32 28, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 0, i32 32, i1 0, i1 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_x2_offset_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+define amdgpu_ps void @buffer_store_x2_offset_merged(<4 x i32> inreg %rsrc, <2 x float> %v1,<2 x float> %v2) {
+  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 0, i32 4, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 0, i32 12, i1 0, i1 0)
+  ret void
+}
+
 declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #0
 declare void @llvm.amdgcn.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i1, i1) #0
 declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #0
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll
index 10bea8ea63b02..69de9555035b9 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.ll
@@ -4,7 +4,7 @@
 declare void @llvm.amdgcn.buffer.wbinvl1() #0
 
 ; GCN-LABEL: {{^}}test_buffer_wbinvl1:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; SI-NEXT: buffer_wbinvl1 ; encoding: [0x00,0x00,0xc4,0xe1,0x00,0x00,0x00,0x00]
 ; VI-NEXT: buffer_wbinvl1 ; encoding: [0x00,0x00,0xf8,0xe0,0x00,0x00,0x00,0x00]
 ; GCN-NEXT: s_endpgm
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll
index fe60d16d90f7a..d1c8f37b3d857 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.sc.ll
@@ -3,7 +3,7 @@
 declare void @llvm.amdgcn.buffer.wbinvl1.sc() #0
 
 ; SI-LABEL: {{^}}test_buffer_wbinvl1_sc:
-; SI-NEXT: ; BB#0:
+; SI-NEXT: ; %bb.0:
 ; SI-NEXT: buffer_wbinvl1_sc ; encoding: [0x00,0x00,0xc0,0xe1,0x00,0x00,0x00,0x00]
 ; SI-NEXT: s_endpgm
 define amdgpu_kernel void @test_buffer_wbinvl1_sc() #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll
index 061c1469ed4d8..4dc938c9b0a28 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll
@@ -4,7 +4,7 @@
 declare void @llvm.amdgcn.buffer.wbinvl1.vol() #0
 
 ; GCN-LABEL: {{^}}test_buffer_wbinvl1_vol:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; CI-NEXT: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xc0,0xe1,0x00,0x00,0x00,0x00]
 ; VI-NEXT: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00]
 ; GCN: s_endpgm
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
index 7179d02fc6ddc..0aa64e2290dc0 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
 
 ; GCN-LABEL: {{^}}s_cvt_pkrtz_v2f16_f32:
 ; GCN-DAG: s_load_dword [[X:s[0-9]+]], s[0:1], 0x{{b|2c}}
@@ -25,7 +25,7 @@ define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(<2 x half> addrspace(1)
 
 ; FIXME: Folds to 0 on gfx9
 ; GCN-LABEL: {{^}}s_cvt_pkrtz_undef_undef:
-; GCN-NEXT: ; BB#0
+; GCN-NEXT: ; %bb.0
 ; SI-NEXT: s_endpgm
 ; VI-NEXT: s_endpgm
 ; GFX9: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}}
@@ -36,8 +36,8 @@ define amdgpu_kernel void @s_cvt_pkrtz_undef_undef(<2 x half> addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
 ; SI: v_cvt_pkrtz_f16_f32_e32 v{{[0-9]+}}, [[A]], [[B]]
 ; GFX89: v_cvt_pkrtz_f16_f32 v{{[0-9]+}}, [[A]], [[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
@@ -54,7 +54,7 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32(<2 x half> addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_reg_imm:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, [[A]], 1.0
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_reg_imm(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -68,7 +68,7 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_reg_imm(<2 x half> addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_imm_reg:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
 ; SI: v_cvt_pkrtz_f16_f32_e32 v{{[0-9]+}}, 1.0, [[A]]
 ; GFX89: v_cvt_pkrtz_f16_f32 v{{[0-9]+}}, 1.0, [[A]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_imm_reg(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr) #0 {
@@ -83,8 +83,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_imm_reg(<2 x half> addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_lo:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -[[A]], [[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -101,8 +101,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo(<2 x half> addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_hi:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, [[A]], -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -119,8 +119,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_hi(<2 x half> addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_lo_hi:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -[[A]], -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -138,8 +138,8 @@ define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_lo_hi(<2 x half> addrspace
 }
 
 ; GCN-LABEL: {{^}}v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi:
-; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
 ; GCN: v_cvt_pkrtz_f16_f32{{(_e64)*}} v{{[0-9]+}}, -|[[A]]|, -[[B]]
 define amdgpu_kernel void @v_cvt_pkrtz_v2f16_f32_fneg_fabs_lo_fneg_hi(<2 x half> addrspace(1)* %out, float addrspace(1)* %a.ptr, float addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
index 3bb5e21d67ac0..9018af3633bd0 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll
@@ -81,8 +81,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}div_fixup_f16_imm_a_imm_b
-; VI:  v_mov_b32_e32 v[[AB_F16:[0-9]+]], 0x4200{{$}}
-; GCN: buffer_load_ushort v[[C_F16:[0-9]+]]
+; VI-DAG:  v_mov_b32_e32 v[[AB_F16:[0-9]+]], 0x4200{{$}}
+; GCN-DAG: buffer_load_ushort v[[C_F16:[0-9]+]]
 ; VI:  v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[AB_F16]], v[[AB_F16]], v[[C_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
@@ -97,8 +97,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}div_fixup_f16_imm_b_imm_c
-; VI:  v_mov_b32_e32 v[[BC_F16:[0-9]+]], 0x4200{{$}}
-; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
+; VI-DAG:  v_mov_b32_e32 v[[BC_F16:[0-9]+]], 0x4200{{$}}
+; GCN-DAG: buffer_load_ushort v[[A_F16:[0-9]+]]
 ; VI:  v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[BC_F16]], v[[BC_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
@@ -113,8 +113,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}div_fixup_f16_imm_a_imm_c
-; VI:  v_mov_b32_e32 v[[AC_F16:[0-9]+]], 0x4200{{$}}
-; GCN: buffer_load_ushort v[[B_F16:[0-9]+]]
+; VI-DAG:  v_mov_b32_e32 v[[AC_F16:[0-9]+]], 0x4200{{$}}
+; GCN-DAG: buffer_load_ushort v[[B_F16:[0-9]+]]
 ; VI:  v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[AC_F16]], v[[B_F16]], v[[AC_F16]]
 ; GCN: buffer_store_short v[[R_F16]]
 ; GCN: s_endpgm
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
index 1b3e09a81e5a0..25b0ad5442104 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
@@ -137,7 +137,6 @@ define amdgpu_kernel void @test_div_fmas_f32_logical_cond_to_vcc(float addrspace
 ; GCN-LABEL: {{^}}test_div_fmas_f32_i1_phi_vcc:
 ; SI: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}}
 ; SI: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[SAVE]], exec, [[SAVE]]
 
 ; SI: buffer_load_dword [[LOAD:v[0-9]+]]
 ; SI: v_cmp_ne_u32_e32 vcc, 0, [[LOAD]]
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
index 0b4f09ac65178..70b01d6977f44 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.div.scale.ll
@@ -1,9 +1,9 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI %s
 
-declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
-declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1) nounwind readnone
-declare { double, i1 } @llvm.amdgcn.div.scale.f64(double, double, i1) nounwind readnone
-declare float @llvm.fabs.f32(float) nounwind readnone
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+declare { float, i1 } @llvm.amdgcn.div.scale.f32(float, float, i1) #1
+declare { double, i1 } @llvm.amdgcn.div.scale.f64(double, double, i1) #1
+declare float @llvm.fabs.f32(float) #1
 
 ; SI-LABEL: {{^}}test_div_scale_f32_1:
 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64
@@ -364,3 +364,82 @@ define amdgpu_kernel void @test_div_scale_f32_fabs_den(float addrspace(1)* %out,
   store float %result0, float addrspace(1)* %out, align 4
   ret void
 }
+
+; SI-LABEL: {{^}}test_div_scale_f32_val_undef_val:
+; SI: v_mov_b32_e32 [[K:v[0-9]+]], 0x41000000
+; SI: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, [[K]], [[K]], [[K]]
+define amdgpu_kernel void @test_div_scale_f32_val_undef_val(float addrspace(1)* %out) #0 {
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.0, float undef, i1 false)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; SI-LABEL: {{^}}test_div_scale_f32_undef_val_val:
+; SI: v_mov_b32_e32 [[K:v[0-9]+]], 0x41000000
+; SI: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, [[K]], [[K]], [[K]]
+define amdgpu_kernel void @test_div_scale_f32_undef_val_val(float addrspace(1)* %out) #0 {
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float 8.0, i1 false)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; SI-LABEL: {{^}}test_div_scale_f32_undef_undef_val:
+; SI-NOT: v0
+; SI: v_div_scale_f32 v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, v0, v0, v0
+define amdgpu_kernel void @test_div_scale_f32_undef_undef_val(float addrspace(1)* %out) #0 {
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float undef, i1 false)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; Undefined selector gets deleted
+; SI-LABEL: {{^}}test_div_scale_f32_val_undef_undef:
+; SI-NOT: v_div_scale
+define amdgpu_kernel void @test_div_scale_f32_val_undef_undef(float addrspace(1)* %out) #0 {
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float 8.0, float undef, i1 undef)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; SI-LABEL: {{^}}test_div_scale_f32_undef_undef_undef:
+; SI-NOT: v_div_scale
+define amdgpu_kernel void @test_div_scale_f32_undef_undef_undef(float addrspace(1)* %out) #0 {
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float undef, float undef, i1 undef)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; SI-LABEL: {{^}}test_div_scale_f32_val_val_undef:
+; SI-NOT: v_div_scale
+define amdgpu_kernel void @test_div_scale_f32_val_val_undef(float addrspace(1)* %out, float addrspace(1)* %in) #0 {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+  %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid
+  %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1
+
+  %a = load volatile float, float addrspace(1)* %gep.0, align 4
+  %b = load volatile float, float addrspace(1)* %gep.1, align 4
+
+  %result = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %a, float %b, i1 undef)
+  %result0 = extractvalue { float, i1 } %result, 0
+  store float %result0, float addrspace(1)* %out, align 4
+  ret void
+}
+
+; SI-LABEL: {{^}}test_div_scale_f64_val_undef_val:
+; SI-DAG: s_mov_b32 s[[K_LO:[0-9]+]], 0{{$}}
+; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0x40200000
+; SI: v_div_scale_f64 v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}}, v[0:1], s{{\[}}[[K_LO]]:[[K_HI]]{{\]}}
+define amdgpu_kernel void @test_div_scale_f64_val_undef_val(double addrspace(1)* %out) #0 {
+  %result = call { double, i1 } @llvm.amdgcn.div.scale.f64(double 8.0, double undef, i1 false)
+  %result0 = extractvalue { double, i1 } %result, 0
+  store double %result0, double addrspace(1)* %out, align 8
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.ll
index 08f286a7f510b..52820237546ea 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.ll
@@ -27,4 +27,27 @@ define amdgpu_kernel void @ds_bpermute_imm_index(i32 addrspace(1)* %out, i32 %ba
   ret void
 }
 
+; CHECK-LABEL: {{^}}ds_bpermute_add_shl:
+; CHECK: ds_bpermute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:4
+; CHECK: s_waitcnt lgkmcnt
+define void @ds_bpermute_add_shl(i32 addrspace(1)* %out, i32 %base_index, i32 %src) nounwind {
+  %index = add i32 %base_index, 1
+  %byte_index = shl i32 %index, 2
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute(i32 %byte_index, i32 %src) #0
+  store i32 %bpermute, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; CHECK-LABEL: {{^}}ds_bpermute_or_shl:
+; CHECK: ds_bpermute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:4
+; CHECK: s_waitcnt lgkmcnt
+define void @ds_bpermute_or_shl(i32 addrspace(1)* %out, i32 %base_index, i32 %src) nounwind {
+  %masked = and i32 %base_index, 62
+  %index = or i32 %masked, 1
+  %byte_index = shl i32 %index, 2
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute(i32 %byte_index, i32 %src) #0
+  store i32 %bpermute, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
 attributes #0 = { nounwind readnone convergent }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.exp.compr.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.exp.compr.ll
index b972ddb8cb770..f08f896fe384f 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.exp.compr.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.exp.compr.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s
 
 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #0
 declare void @llvm.amdgcn.exp.compr.v2i16(i32, i32, <2 x i16>, <2 x i16>, i1, i1) #0
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.f16.ll
index a4ae37b23c5f4..91d1857f306b2 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.fmed3.f16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}test_fmed3_f16:
 ; GCN: v_med3_f16 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.ll
index 87d838727882b..5dec4ad9c1e67 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.ll
@@ -2,6 +2,7 @@
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga -show-mc-encoding -verify-machineinstrs | FileCheck %s --check-prefix=CHECK --check-prefix=VI
 
 ;CHECK-LABEL: {{^}}image_atomic_swap:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_swap v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0x00,0x04,0x00,0x00]
 ;VI: image_atomic_swap v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0x00,0x04,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -13,6 +14,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_swap_v2i32:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_swap v2, v[0:1], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0x00,0x02,0x00,0x00]
 ;VI: image_atomic_swap v2, v[0:1], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0x00,0x02,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -24,6 +26,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_swap_i32:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_swap v1, v0, s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x3c,0xf0,0x00,0x01,0x00,0x00]
 ;VI: image_atomic_swap v1, v0, s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x40,0xf0,0x00,0x01,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -35,6 +38,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_cmpswap:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_cmpswap v[4:5], v[0:3], s[0:7] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x40,0xf0,0x00,0x04,0x00,0x00]
 ;VI: image_atomic_cmpswap v[4:5], v[0:3], s[0:7] dmask:0x3 unorm glc ; encoding: [0x00,0x33,0x44,0xf0,0x00,0x04,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -47,6 +51,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_add:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_add v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x44,0xf0,0x00,0x04,0x00,0x00]
 ;VI: image_atomic_add v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x48,0xf0,0x00,0x04,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -58,6 +63,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_sub:
+;CHECK-NOT: s_waitcnt
 ;SI: image_atomic_sub v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x48,0xf0,0x00,0x04,0x00,0x00]
 ;VI: image_atomic_sub v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x4c,0xf0,0x00,0x04,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
@@ -69,6 +75,7 @@ main_body:
 }
 
 ;CHECK-LABEL: {{^}}image_atomic_unchanged:
+;CHECK-NOT: s_waitcnt
 ;CHECK: image_atomic_smin v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x50,0xf0,0x00,0x04,0x00,0x00]
 ;CHECK: s_waitcnt vmcnt(0)
 ;CHECK: image_atomic_umin v4, v[0:3], s[0:7] dmask:0x1 unorm glc ; encoding: [0x00,0x31,0x54,0xf0,0x00,0x04,0x00,0x00]
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll
index 2e78e2a4c6f57..dfe4aff7bc1c4 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll
@@ -3,6 +3,8 @@
 
 ; GCN-LABEL: {{^}}getlod:
 ; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da
+; GCN: s_waitcnt vmcnt(0)
+; GCN: store_dwordx4
 define amdgpu_kernel void @getlod(<4 x float> addrspace(1)* %out) {
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.f32.v8i32(float undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1)
@@ -12,6 +14,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}getlod_v2:
 ; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da
+; GCN: s_waitcnt vmcnt(0)
+; GCN: store_dwordx4
 define amdgpu_kernel void @getlod_v2(<4 x float> addrspace(1)* %out) {
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1)
@@ -21,6 +25,8 @@ main_body:
 
 ; GCN-LABEL: {{^}}getlod_v4:
 ; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da
+; GCN: s_waitcnt vmcnt(0)
+; GCN: store_dwordx4
 define amdgpu_kernel void @getlod_v4(<4 x float> addrspace(1)* %out) {
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1)
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll
index a289f7b0cfb1e..d9be4a4d01911 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll
@@ -2,6 +2,7 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
 
 ; GCN-LABEL: {{^}}image_load_v4i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps <4 x float> @image_load_v4i32(<8 x i32> inreg %rsrc, <4 x i32> %c) #0 {
@@ -11,6 +12,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_v2i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps <4 x float> @image_load_v2i32(<8 x i32> inreg %rsrc, <2 x i32> %c) #0 {
@@ -20,6 +22,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps <4 x float> @image_load_i32(<8 x i32> inreg %rsrc, i32 %c) #0 {
@@ -29,6 +32,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_mip:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps <4 x float> @image_load_mip(<8 x i32> inreg %rsrc, <4 x i32> %c) #0 {
@@ -38,6 +42,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_1:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load v0, v[0:3], s[0:7] dmask:0x1 unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps float @image_load_1(<8 x i32> inreg %rsrc, <4 x i32> %c) #0 {
@@ -48,6 +53,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_f32_v2i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps float @image_load_f32_v2i32(<8 x i32> inreg %rsrc, <2 x i32> %c) #0 {
@@ -57,6 +63,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_load_v2f32_v4i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_load {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 unorm
 ; GCN: s_waitcnt vmcnt(0)
 define amdgpu_ps <2 x float> @image_load_v2f32_v4i32(<8 x i32> inreg %rsrc, <4 x i32> %c) #0 {
@@ -66,6 +73,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_v4i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm
 define amdgpu_ps void @image_store_v4i32(<8 x i32> inreg %rsrc, <4 x float> %data, <4 x i32> %coords) #0 {
 main_body:
@@ -74,6 +82,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_v2i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm
 define amdgpu_ps void @image_store_v2i32(<8 x i32> inreg %rsrc, <4 x float> %data, <2 x i32> %coords) #0 {
 main_body:
@@ -82,6 +91,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm
 define amdgpu_ps void @image_store_i32(<8 x i32> inreg %rsrc, <4 x float> %data, i32 %coords) #0 {
 main_body:
@@ -90,6 +100,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_f32_i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 unorm
 define amdgpu_ps void @image_store_f32_i32(<8 x i32> inreg %rsrc, float %data, i32 %coords) #0 {
 main_body:
@@ -98,6 +109,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_v2f32_v4i32:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 unorm
 define amdgpu_ps void @image_store_v2f32_v4i32(<8 x i32> inreg %rsrc, <2 x float> %data, <4 x i32> %coords) #0 {
 main_body:
@@ -106,6 +118,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}image_store_mip:
+; GCN-NOT: s_waitcnt
 ; GCN: image_store_mip v[0:3], v[4:7], s[0:7] dmask:0xf unorm
 define amdgpu_ps void @image_store_mip(<8 x i32> inreg %rsrc, <4 x float> %data, <4 x i32> %coords) #0 {
 main_body:
@@ -114,7 +127,10 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}getresinfo:
+; GCN-NOT: s_waitcnt
 ; GCN: image_get_resinfo {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf
+; GCN: s_waitcnt vmcnt(0)
+; GCN: exp
 define amdgpu_ps void @getresinfo() #0 {
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32 undef, <8 x i32> undef, i32 15, i1 false, i1 false, i1 false, i1 false)
@@ -126,6 +142,19 @@ main_body:
   ret void
 }
 
+; GCN-LABEL: {{^}}getresinfo_dmask0:
+; GCN-NOT: image_get_resinfo
+define amdgpu_ps void @getresinfo_dmask0() #0 {
+main_body:
+  %r = call <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32 undef, <8 x i32> undef, i32 0, i1 false, i1 false, i1 false, i1 false)
+  %r0 = extractelement <4 x float> %r, i32 0
+  %r1 = extractelement <4 x float> %r, i32 1
+  %r2 = extractelement <4 x float> %r, i32 2
+  %r3 = extractelement <4 x float> %r, i32 3
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r0, float %r1, float %r2, float %r3, i1 true, i1 true) #0
+  ret void
+}
+
 ; Ideally, the register allocator would avoid the wait here
 ;
 ; GCN-LABEL: {{^}}image_store_wait:
@@ -172,9 +201,10 @@ declare <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32, <8 x i32>, i32,
 declare <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1
 declare <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1
 declare <4 x float> @llvm.amdgcn.image.load.mip.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1
-declare <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #2
 
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
+attributes #2 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll
new file mode 100644
index 0000000000000..e071e6749af52
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.implicitarg.ptr.ll
@@ -0,0 +1,114 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,HSA,HSA-NOENV %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-opencl -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,HSA,HSA-OPENCL %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,MESA %s
+
+; GCN-LABEL: {{^}}kernel_implicitarg_ptr_empty:
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+
+; HSA-NOENV: kernarg_segment_byte_size = 0
+; HSA-OPENCL: kernarg_segment_byte_size = 32
+; MESA: kernarg_segment_byte_size = 16
+
+; HSA: s_load_dword s0, s[4:5], 0x0
+define amdgpu_kernel void @kernel_implicitarg_ptr_empty() #0 {
+  %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %cast = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)*
+  %load = load volatile i32, i32 addrspace(2)* %cast
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_implicitarg_ptr:
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+
+; HSA-NOENV: kernarg_segment_byte_size = 112
+; HSA-OPENCL: kernarg_segment_byte_size = 144
+; MESA: kernarg_segment_byte_size = 464
+
+; HSA: s_load_dword s0, s[4:5], 0x1c
+define amdgpu_kernel void @kernel_implicitarg_ptr([112 x i8]) #0 {
+  %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %cast = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)*
+  %load = load volatile i32, i32 addrspace(2)* %cast
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_implicitarg_ptr:
+; GCN: s_waitcnt
+; GCN-NEXT: s_load_dword s{{[0-9]+}}, s[6:7], 0x0{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @func_implicitarg_ptr() #1 {
+  %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %cast = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)*
+  %load = load volatile i32, i32 addrspace(2)* %cast
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call_implicitarg_ptr_func_empty:
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+; HSA-NOENV: kernarg_segment_byte_size = 0
+; HSA-OPENCL: kernarg_segment_byte_size = 32
+; MESA: kernarg_segment_byte_size = 16
+; GCN: s_mov_b64 s[6:7], s[4:5]
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kernel_call_implicitarg_ptr_func_empty() #0 {
+  call void @func_implicitarg_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call_implicitarg_ptr_func:
+; GCN: enable_sgpr_kernarg_segment_ptr = 1
+; HSA-OPENCL: kernarg_segment_byte_size = 144
+; HSA-NOENV: kernarg_segment_byte_size = 112
+; MESA: kernarg_segment_byte_size = 464
+
+; HSA: s_add_u32 s6, s4, 0x70
+; MESA: s_add_u32 s6, s4, 0x1c0
+
+; GCN: s_addc_u32 s7, s5, 0{{$}}
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kernel_call_implicitarg_ptr_func([112 x i8]) #0 {
+  call void @func_implicitarg_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_implicitarg_ptr_func:
+; GCN-NOT: s6
+; GCN-NOT: s7
+; GCN-NOT: s[6:7]
+define void @func_call_implicitarg_ptr_func() #1 {
+  call void @func_implicitarg_ptr()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_kernarg_implicitarg_ptr:
+; GCN: s_waitcnt
+; GCN: s_load_dword s{{[0-9]+}}, s[6:7], 0x0{{$}}
+; GCN: s_load_dword s{{[0-9]+}}, s[8:9], 0x0{{$}}
+define void @func_kernarg_implicitarg_ptr() #1 {
+  %kernarg.segment.ptr = call i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr()
+  %implicitarg.ptr = call i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr()
+  %cast.kernarg.segment.ptr = bitcast i8 addrspace(2)* %kernarg.segment.ptr to i32 addrspace(2)*
+  %cast.implicitarg = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)*
+  %load0 = load volatile i32, i32 addrspace(2)* %cast.kernarg.segment.ptr
+  %load1 = load volatile i32, i32 addrspace(2)* %cast.implicitarg
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call_kernarg_implicitarg_ptr_func:
+; GCN: s_mov_b64 s[6:7], s[4:5]
+; HSA: s_add_u32 s8, s6, 0x70
+; MESA: s_add_u32 s8, s6, 0x1c0
+; GCN: s_addc_u32 s9, s7, 0
+; GCN: s_swappc_b64
+define amdgpu_kernel void @kernel_call_kernarg_implicitarg_ptr_func([112 x i8]) #0 {
+  call void @func_kernarg_implicitarg_ptr()
+  ret void
+}
+
+declare i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr() #2
+declare i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #2
+
+attributes #0 = { nounwind noinline }
+attributes #1 = { nounwind noinline }
+attributes #2 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
index 617f1f19e360a..5c4dafa38f535 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
@@ -51,7 +51,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_i32_e32 v0, vcc, s0, v0
+; GCN: v_add_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input(i32 inreg %count, i32 %a) {
 main_body:
   call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 19)
@@ -65,7 +65,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_i32_e32 v0, vcc, s0, v0
+; GCN: v_add_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input2(i32 inreg %count, i32 %a) {
 main_body:
   %s = add i32 %a, %count
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
new file mode 100644
index 0000000000000..d6b0628956a02
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
@@ -0,0 +1,241 @@
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI %s
+
+; SI-LABEL: {{^}}gs_const:
+; SI-NOT: v_cmpx
+; SI: s_mov_b64 exec, 0
+define amdgpu_gs void @gs_const() {
+  %tmp = icmp ule i32 0, 3
+  %tmp1 = select i1 %tmp, float 1.000000e+00, float -1.000000e+00
+  %c1 = fcmp oge float %tmp1, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  %tmp2 = icmp ule i32 3, 0
+  %tmp3 = select i1 %tmp2, float 1.000000e+00, float -1.000000e+00
+  %c2 = fcmp oge float %tmp3, 0.0
+  call void @llvm.amdgcn.kill(i1 %c2)
+  ret void
+}
+
+; SI-LABEL: {{^}}vcc_implicit_def:
+; SI-NOT: v_cmp_gt_f32_e32 vcc,
+; SI: v_cmp_gt_f32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], 0, v{{[0-9]+}}
+; SI: v_cmpx_le_f32_e32 vcc, 0, v{{[0-9]+}}
+; SI: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1.0, [[CMP]]
+define amdgpu_ps void @vcc_implicit_def(float %arg13, float %arg14) {
+  %tmp0 = fcmp olt float %arg13, 0.000000e+00
+  %c1 = fcmp oge float %arg14, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  %tmp1 = select i1 %tmp0, float 1.000000e+00, float 0.000000e+00
+  call void @llvm.amdgcn.exp.f32(i32 1, i32 15, float %tmp1, float %tmp1, float %tmp1, float %tmp1, i1 true, i1 true) #0
+  ret void
+}
+
+; SI-LABEL: {{^}}true:
+; SI-NEXT: %bb.
+; SI-NEXT: %bb.
+; SI-NEXT: s_endpgm
+define amdgpu_gs void @true() {
+  call void @llvm.amdgcn.kill(i1 true)
+  ret void
+}
+
+; SI-LABEL: {{^}}false:
+; SI-NOT: v_cmpx
+; SI: s_mov_b64 exec, 0
+define amdgpu_gs void @false() {
+  call void @llvm.amdgcn.kill(i1 false)
+  ret void
+}
+
+; SI-LABEL: {{^}}and:
+; SI: v_cmp_lt_i32
+; SI: v_cmp_lt_i32
+; SI: s_or_b64 s[0:1]
+; SI: s_and_b64 exec, exec, s[0:1]
+define amdgpu_gs void @and(i32 %a, i32 %b, i32 %c, i32 %d) {
+  %c1 = icmp slt i32 %a, %b
+  %c2 = icmp slt i32 %c, %d
+  %x = or i1 %c1, %c2
+  call void @llvm.amdgcn.kill(i1 %x)
+  ret void
+}
+
+; SI-LABEL: {{^}}andn2:
+; SI: v_cmp_lt_i32
+; SI: v_cmp_lt_i32
+; SI: s_xor_b64 s[0:1]
+; SI: s_andn2_b64 exec, exec, s[0:1]
+define amdgpu_gs void @andn2(i32 %a, i32 %b, i32 %c, i32 %d) {
+  %c1 = icmp slt i32 %a, %b
+  %c2 = icmp slt i32 %c, %d
+  %x = xor i1 %c1, %c2
+  %y = xor i1 %x, 1
+  call void @llvm.amdgcn.kill(i1 %y)
+  ret void
+}
+
+; SI-LABEL: {{^}}oeq:
+; SI: v_cmpx_eq_f32
+; SI-NOT: s_and
+define amdgpu_gs void @oeq(float %a) {
+  %c1 = fcmp oeq float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ogt:
+; SI: v_cmpx_lt_f32
+; SI-NOT: s_and
+define amdgpu_gs void @ogt(float %a) {
+  %c1 = fcmp ogt float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}oge:
+; SI: v_cmpx_le_f32
+; SI-NOT: s_and
+define amdgpu_gs void @oge(float %a) {
+  %c1 = fcmp oge float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}olt:
+; SI: v_cmpx_gt_f32
+; SI-NOT: s_and
+define amdgpu_gs void @olt(float %a) {
+  %c1 = fcmp olt float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ole:
+; SI: v_cmpx_ge_f32
+; SI-NOT: s_and
+define amdgpu_gs void @ole(float %a) {
+  %c1 = fcmp ole float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}one:
+; SI: v_cmpx_lg_f32
+; SI-NOT: s_and
+define amdgpu_gs void @one(float %a) {
+  %c1 = fcmp one float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ord:
+; FIXME: This is absolutely unimportant, but we could use the cmpx variant here.
+; SI: v_cmp_o_f32
+define amdgpu_gs void @ord(float %a) {
+  %c1 = fcmp ord float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}uno:
+; FIXME: This is absolutely unimportant, but we could use the cmpx variant here.
+; SI: v_cmp_u_f32
+define amdgpu_gs void @uno(float %a) {
+  %c1 = fcmp uno float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ueq:
+; SI: v_cmpx_nlg_f32
+; SI-NOT: s_and
+define amdgpu_gs void @ueq(float %a) {
+  %c1 = fcmp ueq float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ugt:
+; SI: v_cmpx_nge_f32
+; SI-NOT: s_and
+define amdgpu_gs void @ugt(float %a) {
+  %c1 = fcmp ugt float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}uge:
+; SI: v_cmpx_ngt_f32_e32 vcc, -1.0
+; SI-NOT: s_and
+define amdgpu_gs void @uge(float %a) {
+  %c1 = fcmp uge float %a, -1.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ult:
+; SI: v_cmpx_nle_f32_e32 vcc, -2.0
+; SI-NOT: s_and
+define amdgpu_gs void @ult(float %a) {
+  %c1 = fcmp ult float %a, -2.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}ule:
+; SI: v_cmpx_nlt_f32_e32 vcc, 2.0
+; SI-NOT: s_and
+define amdgpu_gs void @ule(float %a) {
+  %c1 = fcmp ule float %a, 2.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}une:
+; SI: v_cmpx_neq_f32_e32 vcc, 0
+; SI-NOT: s_and
+define amdgpu_gs void @une(float %a) {
+  %c1 = fcmp une float %a, 0.0
+  call void @llvm.amdgcn.kill(i1 %c1)
+  ret void
+}
+
+; SI-LABEL: {{^}}neg_olt:
+; SI: v_cmpx_ngt_f32_e32 vcc, 1.0
+; SI-NOT: s_and
+define amdgpu_gs void @neg_olt(float %a) {
+  %c1 = fcmp olt float %a, 1.0
+  %c2 = xor i1 %c1, 1
+  call void @llvm.amdgcn.kill(i1 %c2)
+  ret void
+}
+
+; SI-LABEL: {{^}}fcmp_x2:
+; FIXME: LLVM should be able to combine these fcmp opcodes.
+; SI: v_cmp_gt_f32
+; SI: v_cndmask_b32
+; SI: v_cmpx_le_f32
+define amdgpu_ps void @fcmp_x2(float %a) #0 {
+  %ogt = fcmp nsz ogt float %a, 2.500000e-01
+  %k = select i1 %ogt, float -1.000000e+00, float 0.000000e+00
+  %c = fcmp nsz oge float %k, 0.000000e+00
+  call void @llvm.amdgcn.kill(i1 %c) #1
+  ret void
+}
+
+; SI-LABEL: {{^}}wqm:
+; SI: v_cmp_neq_f32_e32 vcc, 0
+; SI: s_wqm_b64 s[0:1], vcc
+; SI: s_and_b64 exec, exec, s[0:1]
+define amdgpu_ps void @wqm(float %a) {
+  %c1 = fcmp une float %a, 0.0
+  %c2 = call i1 @llvm.amdgcn.wqm.vote(i1 %c1)
+  call void @llvm.amdgcn.kill(i1 %c2)
+  ret void
+}
+
+declare void @llvm.amdgcn.kill(i1) #0
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
+declare i1 @llvm.amdgcn.wqm.vote(i1)
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
index 144c8f428ab0f..1877cc99ccb80 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
@@ -14,6 +14,24 @@ main_body:
   ret void
 }
 
+; GCN-LABEL: {{^}}mbcnt_lo_known_bits:
+; GCN: v_mbcnt_lo_u32_b32
+; GCN-NOT: and
+define i32 @mbcnt_lo_known_bits(i32 %x, i32 %y) #0 {
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 %x, i32 %y)
+  %mask = and i32 %lo, 63
+  ret i32 %mask
+}
+
+; GCN-LABEL: {{^}}mbcnt_hi_known_bits:
+; GCN: v_mbcnt_hi_u32_b32
+; GCN-NOT: and
+define i32 @mbcnt_hi_known_bits(i32 %x, i32 %y) #0 {
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 %x, i32 %y)
+  %mask = and i32 %hi, 63
+  ret i32 %mask
+}
+
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #0
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
index 8baaad1904067..1ee8320f9c362 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
@@ -5,8 +5,10 @@
 
 ; VI-LABEL: {{^}}dpp_test:
 ; VI: v_mov_b32_e32 v0, s{{[0-9]+}}
+; VI-NOOPT: v_mov_b32_e32 v1, s{{[0-9]+}}
 ; VI: s_nop 1
-; VI: v_mov_b32_dpp v0, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
+; VI-OPT: v_mov_b32_dpp v0, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
+; VI-NOOPT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x01,0x01,0x08,0x11]
 define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in) {
   %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i32 1, i32 1, i1 1) #0
   store i32 %tmp0, i32 addrspace(1)* %out
@@ -14,11 +16,14 @@ define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in) {
 }
 
 ; VI-LABEL: {{^}}dpp_wait_states:
+; VI-NOOPT: v_mov_b32_e32 [[VGPR1:v[0-9]+]], s{{[0-9]+}}
 ; VI: v_mov_b32_e32 [[VGPR0:v[0-9]+]], s{{[0-9]+}}
 ; VI: s_nop 1
-; VI: v_mov_b32_dpp [[VGPR1:v[0-9]+]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; VI-OPT: v_mov_b32_dpp [[VGPR0]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; VI-NOOPT: v_mov_b32_dpp [[VGPR1]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 ; VI: s_nop 1
-; VI: v_mov_b32_dpp v{{[0-9]+}}, [[VGPR1]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; VI-OPT: v_mov_b32_dpp v{{[0-9]+}}, [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
+; VI-NOOPT: v_mov_b32_dpp v{{[0-9]+}}, [[VGPR1]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
 define amdgpu_kernel void @dpp_wait_states(i32 addrspace(1)* %out, i32 %in) {
   %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i32 1, i32 1, i1 1) #0
   %tmp1 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %tmp0, i32 1, i32 1, i32 1, i1 1) #0
@@ -64,3 +69,4 @@ endif:
 declare i32 @llvm.amdgcn.mov.dpp.i32(i32, i32, i32, i32, i1) #0
 
 attributes #0 = { nounwind readnone convergent }
+
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ps.live.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ps.live.ll
index 1c3cba8d3e4fe..3061bd91c9c55 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.ps.live.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ps.live.ll
@@ -1,7 +1,10 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}test1:
-; CHECK: v_cndmask_b32_e64 v0, 0, 1, exec
+; CHECK: s_mov_b64 s[0:1], exec
+; CHECK: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+;
+; Note: The hardware doesn't implement EXEC as src2 for v_cndmask.
 ;
 ; Note: We could generate better code here if we recognized earlier that
 ; there is no WQM use and therefore llvm.amdgcn.ps.live is constant. However,
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 3d815cca5be2d..5a29072a07c33 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -5,7 +5,7 @@
 
 ; GCN-LABEL: {{^}}test_barrier:
 ; GFX8: buffer_store_dword
-; GFX9: flat_store_dword
+; GFX9: global_store_dword
 ; NOAUTO: s_waitcnt
 ; AUTO-NOT: s_waitcnt
 ; GCN: s_barrier
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll
index 224b2ed72e3bd..b7fb96a2d1a59 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.ll
@@ -5,7 +5,7 @@ declare void @llvm.amdgcn.s.dcache.inv() #0
 declare void @llvm.amdgcn.s.waitcnt(i32) #0
 
 ; GCN-LABEL: {{^}}test_s_dcache_inv:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; SI-NEXT: s_dcache_inv ; encoding: [0x00,0x00,0xc0,0xc7]
 ; VI-NEXT: s_dcache_inv ; encoding: [0x00,0x00,0x80,0xc0,0x00,0x00,0x00,0x00]
 ; GCN-NEXT: s_endpgm
@@ -15,7 +15,7 @@ define amdgpu_kernel void @test_s_dcache_inv() #0 {
 }
 
 ; GCN-LABEL: {{^}}test_s_dcache_inv_insert_wait:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; GCN: s_dcache_inv
 ; GCN: s_waitcnt lgkmcnt(0) ; encoding
 define amdgpu_kernel void @test_s_dcache_inv_insert_wait() #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll
index f96d5db5794ac..e8a363adde734 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.inv.vol.ll
@@ -5,7 +5,7 @@ declare void @llvm.amdgcn.s.dcache.inv.vol() #0
 declare void @llvm.amdgcn.s.waitcnt(i32) #0
 
 ; GCN-LABEL: {{^}}test_s_dcache_inv_vol:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; CI-NEXT: s_dcache_inv_vol ; encoding: [0x00,0x00,0x40,0xc7]
 ; VI-NEXT: s_dcache_inv_vol ; encoding: [0x00,0x00,0x88,0xc0,0x00,0x00,0x00,0x00]
 ; GCN-NEXT: s_endpgm
@@ -15,7 +15,7 @@ define amdgpu_kernel void @test_s_dcache_inv_vol() #0 {
 }
 
 ; GCN-LABEL: {{^}}test_s_dcache_inv_vol_insert_wait:
-; GCN-NEXT: ; BB#0:
+; GCN-NEXT: ; %bb.0:
 ; GCN-NEXT: s_dcache_inv_vol
 ; GCN: s_waitcnt lgkmcnt(0) ; encoding
 define amdgpu_kernel void @test_s_dcache_inv_vol_insert_wait() #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll
index 99b6513504395..254a0fae3c3b7 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.ll
@@ -4,7 +4,7 @@ declare void @llvm.amdgcn.s.dcache.wb() #0
 declare void @llvm.amdgcn.s.waitcnt(i32) #0
 
 ; VI-LABEL: {{^}}test_s_dcache_wb:
-; VI-NEXT: ; BB#0:
+; VI-NEXT: ; %bb.0:
 ; VI-NEXT: s_dcache_wb ; encoding: [0x00,0x00,0x84,0xc0,0x00,0x00,0x00,0x00]
 ; VI-NEXT: s_endpgm
 define amdgpu_kernel void @test_s_dcache_wb() #0 {
@@ -13,7 +13,7 @@ define amdgpu_kernel void @test_s_dcache_wb() #0 {
 }
 
 ; VI-LABEL: {{^}}test_s_dcache_wb_insert_wait:
-; VI-NEXT: ; BB#0:
+; VI-NEXT: ; %bb.0:
 ; VI-NEXT: s_dcache_wb
 ; VI: s_waitcnt lgkmcnt(0) ; encoding
 define amdgpu_kernel void @test_s_dcache_wb_insert_wait() #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll
index 844fcecdb48bf..929cd1c5f0bbd 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.dcache.wb.vol.ll
@@ -4,7 +4,7 @@ declare void @llvm.amdgcn.s.dcache.wb.vol() #0
 declare void @llvm.amdgcn.s.waitcnt(i32) #0
 
 ; VI-LABEL: {{^}}test_s_dcache_wb_vol:
-; VI-NEXT: ; BB#0:
+; VI-NEXT: ; %bb.0:
 ; VI-NEXT: s_dcache_wb_vol ; encoding: [0x00,0x00,0x8c,0xc0,0x00,0x00,0x00,0x00]
 ; VI-NEXT: s_endpgm
 define amdgpu_kernel void @test_s_dcache_wb_vol() #0 {
@@ -13,7 +13,7 @@ define amdgpu_kernel void @test_s_dcache_wb_vol() #0 {
 }
 
 ; VI-LABEL: {{^}}test_s_dcache_wb_vol_insert_wait:
-; VI-NEXT: ; BB#0:
+; VI-NEXT: ; %bb.0:
 ; VI-NEXT: s_dcache_wb_vol
 ; VI: s_waitcnt lgkmcnt(0) ; encoding
 define amdgpu_kernel void @test_s_dcache_wb_vol_insert_wait() #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll
index a466671d8c552..f6c2cb44c9938 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll
@@ -2,6 +2,7 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}test1:
+; CHECK-NOT: s_waitcnt
 ; CHECK: image_store
 ; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0){{$}}
 ; CHECK-NEXT: image_store
@@ -17,6 +18,7 @@ define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <4 x float> %d0, <4 x float>
 ; emitted as late as possible.
 ;
 ; CHECK-LABEL: {{^}}test2:
+; CHECK-NOT: s_waitcnt
 ; CHECK: image_load
 ; CHECK-NEXT: s_waitcnt
 ; CHECK: s_waitcnt vmcnt(0){{$}}
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sbfe.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sbfe.ll
index 871b8c4f99b99..b3e1c86d94796 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.sbfe.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sbfe.ll
@@ -396,7 +396,7 @@ define amdgpu_kernel void @bfe_sext_in_reg_i24(i32 addrspace(1)* %out, i32 addrs
 ; GCN: buffer_load_dword [[LOAD:v[0-9]+]]
 ; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[LOAD]], 1, 16
 ; GCN: v_lshrrev_b32_e32 [[TMP0:v[0-9]+]], 31, [[BFE]]
-; GCN: v_add_i32_e32 [[TMP1:v[0-9]+]], vcc, [[TMP0]], [[BFE]]
+; GCN: v_add_{{[iu]}}32_e32 [[TMP1:v[0-9]+]], vcc, [[TMP0]], [[BFE]]
 ; GCN: v_ashrrev_i32_e32 [[TMP2:v[0-9]+]], 1, [[TMP1]]
 ; GCN: buffer_store_dword [[TMP2]]
 define amdgpu_kernel void @simplify_demanded_bfe_sdiv(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
new file mode 100644
index 0000000000000..648a90afbe5c7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
@@ -0,0 +1,29 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s
+
+
+; GCN-LABEL: {{^}}set_inactive:
+; GCN: s_not_b64 exec, exec
+; GCN: v_mov_b32_e32 {{v[0-9]+}}, 42
+; GCN: s_not_b64 exec, exec
+define amdgpu_kernel void @set_inactive(i32 addrspace(1)* %out, i32 %in) {
+  %tmp = call i32 @llvm.amdgcn.set.inactive.i32(i32 %in, i32 42) #0
+  store i32 %tmp, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}set_inactive_64:
+; GCN: s_not_b64 exec, exec
+; GCN: v_mov_b32_e32 {{v[0-9]+}}, 0
+; GCN: v_mov_b32_e32 {{v[0-9]+}}, 0
+; GCN: s_not_b64 exec, exec
+define amdgpu_kernel void @set_inactive_64(i64 addrspace(1)* %out, i64 %in) {
+  %tmp = call i64 @llvm.amdgcn.set.inactive.i64(i64 %in, i64 0) #0
+  store i64 %tmp, i64 addrspace(1)* %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #0
+declare i64 @llvm.amdgcn.set.inactive.i64(i64, i64) #0
+
+attributes #0 = { convergent readnone }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
index 68fd08f778c43..62bd684d5411c 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
@@ -65,7 +65,7 @@ define amdgpu_kernel void @bfe_u32_zextload_i8(i32 addrspace(1)* %out, i8 addrsp
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i8:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN-NEXT: v_and_b32_e32
 ; FIXME: Should be using s_add_i32
 ; GCN-NOT: {{[^@]}}bfe
@@ -81,7 +81,7 @@ define amdgpu_kernel void @bfe_u32_zext_in_reg_i8(i32 addrspace(1)* %out, i32 ad
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i16:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN-NEXT: v_and_b32_e32
 ; GCN-NOT: {{[^@]}}bfe
 ; GCN: s_endpgm
@@ -96,7 +96,7 @@ define amdgpu_kernel void @bfe_u32_zext_in_reg_i16(i32 addrspace(1)* %out, i32 a
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i8_offset_1:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN: bfe
 ; GCN: s_endpgm
 define amdgpu_kernel void @bfe_u32_zext_in_reg_i8_offset_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
@@ -110,7 +110,7 @@ define amdgpu_kernel void @bfe_u32_zext_in_reg_i8_offset_1(i32 addrspace(1)* %ou
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i8_offset_3:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN-NEXT: v_and_b32_e32 {{v[0-9]+}}, 0xf8
 ; GCN-NEXT: bfe
 ; GCN: s_endpgm
@@ -125,7 +125,7 @@ define amdgpu_kernel void @bfe_u32_zext_in_reg_i8_offset_3(i32 addrspace(1)* %ou
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i8_offset_7:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN-NEXT: v_and_b32_e32 {{v[0-9]+}}, 0x80
 ; GCN-NEXT: bfe
 ; GCN: s_endpgm
@@ -140,7 +140,7 @@ define amdgpu_kernel void @bfe_u32_zext_in_reg_i8_offset_7(i32 addrspace(1)* %ou
 
 ; GCN-LABEL: {{^}}bfe_u32_zext_in_reg_i16_offset_8:
 ; GCN: buffer_load_dword
-; GCN: v_add_i32
+; GCN: v_add_{{[iu]}}32
 ; GCN-NEXT: bfe
 ; GCN: s_endpgm
 define amdgpu_kernel void @bfe_u32_zext_in_reg_i16_offset_8(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
new file mode 100644
index 0000000000000..4d1dbf514a105
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll
@@ -0,0 +1,17 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=VI-OPT %s
+; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI -check-prefix=VI-NOOPT %s
+
+; VI-LABEL: {{^}}dpp_test:
+; VI: v_mov_b32_e32 v0, s{{[0-9]+}}
+; VI: v_mov_b32_e32 v1, s{{[0-9]+}}
+; VI: s_nop 1
+; VI: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x01,0x01,0x08,0x11]
+define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in1, i32 %in2) {
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %in1, i32 %in2, i32 1, i32 1, i32 1, i1 1) #0
+  store i32 %tmp0, i32 addrspace(1)* %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0
+
+attributes #0 = { nounwind readnone convergent }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll
new file mode 100644
index 0000000000000..1946e6a361867
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.wqm.vote.ll
@@ -0,0 +1,52 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=CHECK %s
+
+;CHECK-LABEL: {{^}}ret:
+;CHECK: v_cmp_eq_u32_e32 [[CMP:[^,]+]], v0, v1
+;CHECK: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
+;CHECK: v_cndmask_b32_e64 v0, 0, 1.0, [[WQM]]
+define amdgpu_ps float @ret(i32 %v0, i32 %v1) #1 {
+main_body:
+  %c = icmp eq i32 %v0, %v1
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 %c)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+;CHECK-LABEL: {{^}}true:
+;CHECK: s_wqm_b64
+define amdgpu_ps float @true() #1 {
+main_body:
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 true)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+;CHECK-LABEL: {{^}}false:
+;CHECK: s_wqm_b64
+define amdgpu_ps float @false() #1 {
+main_body:
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 false)
+  %r = select i1 %w, float 1.0, float 0.0
+  ret float %r
+}
+
+;CHECK-LABEL: {{^}}kill:
+;CHECK: v_cmp_eq_u32_e32 [[CMP:[^,]+]], v0, v1
+;CHECK: s_wqm_b64 [[WQM:[^,]+]], [[CMP]]
+;FIXME: This could just be: s_and_b64 exec, exec, [[WQM]]
+;CHECK: v_cndmask_b32_e64 [[KILL:[^,]+]], -1.0, 1.0, [[WQM]]
+;CHECK: v_cmpx_le_f32_e32 {{[^,]+}}, 0, [[KILL]]
+;CHECK: s_endpgm
+define amdgpu_ps void @kill(i32 %v0, i32 %v1) #1 {
+main_body:
+  %c = icmp eq i32 %v0, %v1
+  %w = call i1 @llvm.amdgcn.wqm.vote(i1 %c)
+  %r = select i1 %w, float 1.0, float -1.0
+  call void @llvm.AMDGPU.kill(float %r)
+  ret void
+}
+
+declare void @llvm.AMDGPU.kill(float) #1
+declare i1 @llvm.amdgcn.wqm.vote(i1)
+
+attributes #1 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/llvm.ceil.f16.ll b/test/CodeGen/AMDGPU/llvm.ceil.f16.ll
index 071f2a6de4cd0..0f3cf79e257b2 100644
--- a/test/CodeGen/AMDGPU/llvm.ceil.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.ceil.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.ceil.f16(half %a)
 declare <2 x half> @llvm.ceil.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.cos.f16.ll b/test/CodeGen/AMDGPU/llvm.cos.f16.ll
index 8931de63e74ba..16f167ae9e389 100644
--- a/test/CodeGen/AMDGPU/llvm.cos.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.cos.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.cos.f16(half %a)
 declare <2 x half> @llvm.cos.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.dbg.value.ll b/test/CodeGen/AMDGPU/llvm.dbg.value.ll
index c4a76de5989c2..ace859c957527 100644
--- a/test/CodeGen/AMDGPU/llvm.dbg.value.ll
+++ b/test/CodeGen/AMDGPU/llvm.dbg.value.ll
@@ -1,22 +1,37 @@
-; RUN: llc -O0 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -verify-machineinstrs -mattr=-flat-for-global < %s | FileCheck %s
+; RUN: llc -O0 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,NOOPT %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,OPT %s
 
-; CHECK-LABEL: {{^}}test_debug_value:
-; CHECK: s_load_dwordx2 s[4:5]
+; GCN-LABEL: {{^}}test_debug_value:
+; NOOPT: s_load_dwordx2 s[4:5]
 
 ; FIXME: Why is the SGPR4_SGPR5 reference being removed from DBG_VALUE?
-; CHECK: ; kill: %SGPR4_SGPR5<def> %SGPR4_SGPR5<kill>
-; CHECK-NEXT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- undef
+; NOOPT: ; kill: def %sgpr8_sgpr9 killed %sgpr4_sgpr5
+; NOOPT-NEXT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- undef
 
-; CHECK: buffer_store_dword
-; CHECK: s_endpgm
+; GCN: flat_store_dword
+; GCN: s_endpgm
 define amdgpu_kernel void @test_debug_value(i32 addrspace(1)* nocapture %globalptr_arg) #0 !dbg !4 {
 entry:
-  tail call void @llvm.dbg.value(metadata i32 addrspace(1)* %globalptr_arg, i64 0, metadata !10, metadata !13), !dbg !14
+  tail call void @llvm.dbg.value(metadata i32 addrspace(1)* %globalptr_arg, metadata !10, metadata !13), !dbg !14
   store i32 123, i32 addrspace(1)* %globalptr_arg, align 4
   ret void
 }
 
-declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+; Check for infinite loop in some cases with dbg_value in
+; SIOptimizeExecMaskingPreRA (somehow related to undef argument).
+
+; GCN-LABEL: {{^}}only_undef_dbg_value:
+; NOOPT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- [DW_OP_constu 1, DW_OP_swap, DW_OP_xderef] undef
+; NOOPT-NEXT: s_endpgm
+
+; OPT: s_endpgm
+define amdgpu_kernel void @only_undef_dbg_value() #1 {
+bb:
+  call void @llvm.dbg.value(metadata <4 x float> undef, metadata !10, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)) #2, !dbg !14
+  ret void
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
 attributes #0 = { nounwind  }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/llvm.exp2.f16.ll b/test/CodeGen/AMDGPU/llvm.exp2.f16.ll
index 4e96a76197160..5764224ccd6ae 100644
--- a/test/CodeGen/AMDGPU/llvm.exp2.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.exp2.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.exp2.f16(half %a)
 declare <2 x half> @llvm.exp2.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.floor.f16.ll b/test/CodeGen/AMDGPU/llvm.floor.f16.ll
index 74d1e694ffbe2..907fd488fa728 100644
--- a/test/CodeGen/AMDGPU/llvm.floor.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.floor.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.floor.f16(half %a)
 declare <2 x half> @llvm.floor.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.fma.f16.ll b/test/CodeGen/AMDGPU/llvm.fma.f16.ll
index a379b18ffb8b6..bfcce66ac1d01 100644
--- a/test/CodeGen/AMDGPU/llvm.fma.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.fma.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.fma.f16(half %a, half %b, half %c)
 declare <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
@@ -253,7 +253,7 @@ define amdgpu_kernel void @fma_v2f16_imm_b(
 ; VI-DAG:  v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16]]
 
 ; GCN-NOT: and
-; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
+; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_1]]
 ; GCN: buffer_store_dword v[[R_V2_F16]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @fma_v2f16_imm_c(
diff --git a/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll b/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
index 2d4fe08d8bde8..478658c3acc88 100644
--- a/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll
@@ -1,7 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SI-FLUSH %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=VI-FLUSH %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=+fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SI-DENORM %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=VI-DENORM %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-FLUSH %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-FLUSH %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -mattr=+fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=SI-DENORM %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=+fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=VI-DENORM %s
 
 declare half @llvm.fmuladd.f16(half %a, half %b, half %c)
 declare <2 x half> @llvm.fmuladd.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)
@@ -70,7 +70,7 @@ define amdgpu_kernel void @fmuladd_f16_imm_a(
 ; GCN: buffer_load_ushort v[[C_F16:[0-9]+]]
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
 ; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
-; SI:  v_mac_f32_e32 v[[C_F32]], 0x40400000, v[[B_F32]]
+; SI:  v_mac_f32_e32 v[[C_F32]], 0x40400000, v[[A_F32]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[C_F32]]
 ; SI:  buffer_store_short v[[R_F16]]
 
@@ -95,8 +95,9 @@ define amdgpu_kernel void @fmuladd_f16_imm_b(
 }
 
 ; GCN-LABEL: {{^}}fmuladd_v2f16
+; VI:  buffer_load_dword v[[B_V2_F16:[0-9]+]]
 ; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]]
-; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]]
+; SI:  buffer_load_dword v[[B_V2_F16:[0-9]+]]
 ; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]]
 
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
@@ -124,11 +125,11 @@ define amdgpu_kernel void @fmuladd_f16_imm_b(
 ; VI-FLUSH-NOT: v_and_b32
 ; VI-FLUSH:     v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], v[[R_F16_HI]]
 
-; VI-DENORM: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
-; VI-DENORM: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
-; VI-DENORM: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
-; VI-DENORM-DAG: v_fma_f16 v[[RES0:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_V2_F16]]
-; VI-DENORM-DAG: v_fma_f16 v[[RES1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16_1]]
+; VI-DENORM-DAG: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
+; VI-DENORM-DAG: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
+; VI-DENORM-DAG: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]]
+; VI-DENORM-DAG: v_fma_f16 v[[RES0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]], v[[C_V2_F16]]
+; VI-DENORM-DAG: v_fma_f16 v[[RES1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]], v[[C_F16_1]]
 ; VI-DENORM-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[RES1]]
 ; VI-DENORM-NOT: v_and_b32
 ; VI-DENORM: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[RES0]], v[[R_F16_HI]]
diff --git a/test/CodeGen/AMDGPU/llvm.log.f16.ll b/test/CodeGen/AMDGPU/llvm.log.f16.ll
new file mode 100644
index 0000000000000..d64c0bba41227
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.log.f16.ll
@@ -0,0 +1,71 @@
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SIVI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=SIVI -check-prefix=VIGFX9 -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=VIGFX9 -check-prefix=FUNC %s
+
+declare half @llvm.log.f16(half %a)
+declare <2 x half> @llvm.log.v2f16(<2 x half> %a)
+
+; FUNC-LABEL: {{^}}log_f16
+; SI:     buffer_load_ushort v[[A_F16_0:[0-9]+]]
+; VI:     flat_load_ushort v[[A_F16_0:[0-9]+]]
+; GFX9:   global_load_ushort v[[A_F16_0:[0-9]+]]
+; SI:     v_mov_b32_e32 v[[A_F32_1:[0-9]+]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
+; SI:     v_mul_f32_e32 v[[R_F32_1:[0-9]+]], 0x3f317218, v[[R_F32_0]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_1]]
+; VIGFX9: v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
+; VIGFX9: v_mul_f16_e32 v[[R_F16_0]], 0x398c, v[[R_F16_0]]
+; SI:     buffer_store_short v[[R_F16_0]], v{{\[[0-9]+:[0-9]+\]}}
+; VI:     flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[R_F16_0]]
+; GFX9:   global_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[R_F16_0]]
+define void @log_f16(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a) {
+entry:
+  %a.val = load half, half addrspace(1)* %a
+  %r.val = call half @llvm.log.f16(half %a.val)
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}
+
+; FUNC-LABEL: {{^}}log_v2f16
+; SI:     buffer_load_dword v[[A_F16_0:[0-9]+]]
+; VI:     flat_load_dword v[[A_F16_0:[0-9]+]]
+; GFX9:   global_load_dword v[[A_F16_0:[0-9]+]]
+; SI:     v_mov_b32_e32 v[[A_F32_2:[0-9]+]], 0x3f317218
+; VIGFX9: v_mov_b32_e32 v[[A_F32_2:[0-9]+]], 0x398c
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
+; SI:     v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
+; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
+; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], v[[R_F32_1]], v[[A_F32_2]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
+; SI:     v_mul_f32_e32 v[[R_F32_5:[0-9]+]], v[[R_F32_0]], v[[A_F32_2]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_5]]
+; GFX9:   v_log_f16_e32 v[[R_F16_2:[0-9]+]], v[[A_F16_0]]
+; VIGFX9: v_log_f16_sdwa v[[R_F16_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI:     v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
+; VI:     v_mul_f16_sdwa v[[R_F16_2:[0-9]+]], v[[R_F16_1]], v[[A_F32_2]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9:   v_mul_f16_e32 v[[R_F32_3:[0-9]+]], v[[R_F16_2]], v[[A_F32_2]]
+; VIGFX9: v_mul_f16_e32 v[[R_F32_2:[0-9]+]], v[[R_F16_0]], v[[A_F32_2]]
+; SI:     v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_0]]
+; SI-NOT: v_and_b32_e32
+; SI:     v_or_b32_e32 v[[R_F32_5:[0-9]+]], v[[R_F16_1]], v[[R_F16_0]]
+; VI-NOT: v_and_b32_e32
+; VI:     v_or_b32_e32 v[[R_F32_5:[0-9]+]], v[[R_F16_0]], v[[R_F16_2]]
+; GFX9:   v_and_b32_e32 v[[R_F32_4:[0-9]+]], 0xffff, v[[R_F32_3]]
+; GFX9:   v_lshl_or_b32 v[[R_F32_5:[0-9]+]], v[[R_F32_2]], 16, v[[R_F32_4]]
+; SI:     buffer_store_dword v[[R_F32_5]]
+; VI:     flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[R_F32_5]]
+; GFX9:   global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[R_F32_5]]
+define void @log_v2f16(
+    <2 x half> addrspace(1)* %r,
+    <2 x half> addrspace(1)* %a) {
+entry:
+  %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
+  %r.val = call <2 x half> @llvm.log.v2f16(<2 x half> %a.val)
+  store <2 x half> %r.val, <2 x half> addrspace(1)* %r
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/llvm.log.ll b/test/CodeGen/AMDGPU/llvm.log.ll
new file mode 100644
index 0000000000000..8018bf277cf95
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.log.ll
@@ -0,0 +1,89 @@
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC  %s
+; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 --check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+
+; FUNC-LABEL: {{^}}test:
+; EG: LOG_IEEE
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x3f317218, v{{[0-9]+}}
+define void @test(float addrspace(1)* %out, float %in) {
+entry:
+   %res = call float @llvm.log.f32(float %in)
+   store float %res, float addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}testv2:
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; FIXME: We should be able to merge these packets together on Cayman so we
+; have a maximum of 4 instructions.
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3f317218
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GFX8: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3f317218
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+define void @testv2(<2 x float> addrspace(1)* %out, <2 x float> %in) {
+entry:
+  %res = call <2 x float> @llvm.log.v2f32(<2 x float> %in)
+  store <2 x float> %res, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}testv4:
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; FIXME: We should be able to merge these packets together on Cayman so we
+; have a maximum of 4 instructions.
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3f317218
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GFX8: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3f317218
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+define void @testv4(<4 x float> addrspace(1)* %out, <4 x float> %in) {
+entry:
+  %res = call <4 x float> @llvm.log.v4f32(<4 x float> %in)
+  store <4 x float> %res, <4 x float> addrspace(1)* %out
+  ret void
+}
+
+declare float @llvm.log.f32(float) readnone
+declare <2 x float> @llvm.log.v2f32(<2 x float>) readnone
+declare <4 x float> @llvm.log.v4f32(<4 x float>) readnone
diff --git a/test/CodeGen/AMDGPU/llvm.log10.f16.ll b/test/CodeGen/AMDGPU/llvm.log10.f16.ll
new file mode 100644
index 0000000000000..70cac1d2539d1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.log10.f16.ll
@@ -0,0 +1,71 @@
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SIVI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=SIVI -check-prefix=VIGFX9 -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -check-prefix=VIGFX9 -check-prefix=FUNC %s
+
+declare half @llvm.log10.f16(half %a)
+declare <2 x half> @llvm.log10.v2f16(<2 x half> %a)
+
+; GCN-LABEL: {{^}}log10_f16
+; SI:     buffer_load_ushort v[[A_F16_0:[0-9]+]]
+; VI:     flat_load_ushort v[[A_F16_0:[0-9]+]]
+; GFX9:   global_load_ushort v[[A_F16_0:[0-9]+]]
+; SI:     v_mov_b32_e32 v[[A_F32_1:[0-9]+]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
+; SI:     v_mul_f32_e32 v[[R_F32_1:[0-9]+]], 0x3e9a209a, v[[R_F32_0]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_1]]
+; VIGFX9: v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
+; VIGFX9: v_mul_f16_e32 v[[R_F16_0]], 0x34d1, v[[R_F16_0]]
+; SI:     buffer_store_short v[[R_F16_0]], v{{\[[0-9]+:[0-9]+\]}}
+; VI:     flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[R_F16_0]]
+; GFX9:   global_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[R_F16_0]]
+define void @log10_f16(
+    half addrspace(1)* %r,
+    half addrspace(1)* %a) {
+entry:
+  %a.val = load half, half addrspace(1)* %a
+  %r.val = call half @llvm.log10.f16(half %a.val)
+  store half %r.val, half addrspace(1)* %r
+  ret void
+}
+
+; GCN-LABEL: {{^}}log10_v2f16
+; SI:     buffer_load_dword v[[A_F16_0:[0-9]+]]
+; VI:     flat_load_dword v[[A_F16_0:[0-9]+]]
+; GFX9:   global_load_dword v[[A_F16_0:[0-9]+]]
+; SI:     v_mov_b32_e32 v[[A_F32_2:[0-9]+]], 0x3e9a209a
+; VIGFX9: v_mov_b32_e32 v[[A_F32_2:[0-9]+]], 0x34d1
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
+; SI:     v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
+; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
+; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
+; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], v[[R_F32_1]], v[[A_F32_2]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
+; SI:     v_mul_f32_e32 v[[R_F32_5:[0-9]+]], v[[R_F32_0]], v[[A_F32_2]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_5]]
+; GFX9:   v_log_f16_e32 v[[R_F16_2:[0-9]+]], v[[A_F16_0]]
+; VIGFX9: v_log_f16_sdwa v[[R_F16_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI:     v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
+; VI:     v_mul_f16_sdwa v[[R_F16_2:[0-9]+]], v[[R_F16_1]], v[[A_F32_2]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9:   v_mul_f16_e32 v[[R_F32_3:[0-9]+]], v[[R_F16_2]], v[[A_F32_2]]
+; VIGFX9: v_mul_f16_e32 v[[R_F32_2:[0-9]+]], v[[R_F16_0]], v[[A_F32_2]]
+; SI:     v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_0]]
+; SI-NOT: v_and_b32_e32
+; SI:     v_or_b32_e32 v[[R_F32_5:[0-9]+]], v[[R_F16_1]], v[[R_F16_0]]
+; VI-NOT: v_and_b32_e32
+; VI:     v_or_b32_e32 v[[R_F32_5:[0-9]+]], v[[R_F16_0]], v[[R_F16_2]]
+; GFX9:   v_and_b32_e32 v[[R_F32_4:[0-9]+]], 0xffff, v[[R_F32_3]]
+; GFX9:   v_lshl_or_b32 v[[R_F32_5:[0-9]+]], v[[R_F32_2]], 16, v[[R_F32_4]]
+; SI:     buffer_store_dword v[[R_F32_5]]
+; VI:     flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[R_F32_5]]
+; GFX9:   global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[R_F32_5]]
+define void @log10_v2f16(
+    <2 x half> addrspace(1)* %r,
+    <2 x half> addrspace(1)* %a) {
+entry:
+  %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
+  %r.val = call <2 x half> @llvm.log10.v2f16(<2 x half> %a.val)
+  store <2 x half> %r.val, <2 x half> addrspace(1)* %r
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/llvm.log10.ll b/test/CodeGen/AMDGPU/llvm.log10.ll
new file mode 100644
index 0000000000000..1812f47c8559e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -0,0 +1,89 @@
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC  %s
+; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=GFX8 -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+
+; FUNC-LABEL: {{^}}test:
+; EG: LOG_IEEE
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x3e9a209a, v{{[0-9]+}}
+define void @test(float addrspace(1)* %out, float %in) {
+entry:
+   %res = call float @llvm.log10.f32(float %in)
+   store float %res, float addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}testv2:
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; FIXME: We should be able to merge these packets together on Cayman so we
+; have a maximum of 4 instructions.
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3e9a209a
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GFX8: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3e9a209a
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+define void @testv2(<2 x float> addrspace(1)* %out, <2 x float> %in) {
+entry:
+  %res = call <2 x float> @llvm.log10.v2f32(<2 x float> %in)
+  store <2 x float> %res, <2 x float> addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}testv4:
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; EG: LOG_IEEE
+; FIXME: We should be able to merge these packets together on Cayman so we
+; have a maximum of 4 instructions.
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}} (MASKED)
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; CM-DAG: LOG_IEEE T{{[0-9]+\.[XYZW]}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3e9a209a
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_log_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}
+; GFX8: v_mov_b32_e32 v[[R_F32_LOG_CONST:[0-9]+]], 0x3e9a209a
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+; GCN: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[R_F32_LOG_CONST]]
+define void @testv4(<4 x float> addrspace(1)* %out, <4 x float> %in) {
+entry:
+  %res = call <4 x float> @llvm.log10.v4f32(<4 x float> %in)
+  store <4 x float> %res, <4 x float> addrspace(1)* %out
+  ret void
+}
+
+declare float @llvm.log10.f32(float) readnone
+declare <2 x float> @llvm.log10.v2f32(<2 x float>) readnone
+declare <4 x float> @llvm.log10.v4f32(<4 x float>) readnone
diff --git a/test/CodeGen/AMDGPU/llvm.log2.f16.ll b/test/CodeGen/AMDGPU/llvm.log2.f16.ll
index 277195c532086..7228c4044c7a8 100644
--- a/test/CodeGen/AMDGPU/llvm.log2.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.log2.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.log2.f16(half %a)
 declare <2 x half> @llvm.log2.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.log2.ll b/test/CodeGen/AMDGPU/llvm.log2.ll
index b9d593e43f32c..15138cff00d1c 100644
--- a/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -1,6 +1,6 @@
 ;RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC
 ;RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=CM --check-prefix=FUNC
-;RUN: llc < %s -march=amdgcn -mcpu=SI | FileCheck %s --check-prefix=SI --check-prefix=FUNC
+;RUN: llc < %s -march=amdgcn -mcpu=tahiti | FileCheck %s --check-prefix=SI --check-prefix=FUNC
 ;RUN: llc < %s -march=amdgcn -mcpu=tonga | FileCheck %s --check-prefix=SI --check-prefix=FUNC
 
 ;FUNC-LABEL: {{^}}test:
diff --git a/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll b/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
index c72716439a761..aa085a71affbd 100644
--- a/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.maxnum.f16(half %a, half %b)
 declare <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> %b)
@@ -137,8 +137,9 @@ entry:
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]]
 ; SI:  v_max_f32_e32 v[[R_F32_1:[0-9]+]], 0x40400000, v[[A_F32_1]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
+
 ; VI-DAG:  v_mov_b32_e32 [[CONST3:v[0-9]+]], 0x4200
-; VI-DAG:  v_max_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[B_V2_F16]], [[CONST3]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-DAG:  v_max_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[A_V2_F16]], [[CONST3]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; VI-DAG:  v_max_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]]
 
 ; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
diff --git a/test/CodeGen/AMDGPU/llvm.memcpy.ll b/test/CodeGen/AMDGPU/llvm.memcpy.ll
index 7b4db55155eb3..4068c020e7058 100644
--- a/test/CodeGen/AMDGPU/llvm.memcpy.ll
+++ b/test/CodeGen/AMDGPU/llvm.memcpy.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 declare void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* nocapture, i8 addrspace(3)* nocapture, i32, i32, i1) nounwind
diff --git a/test/CodeGen/AMDGPU/llvm.minnum.f16.ll b/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
index 0e93acc27dc5b..5ab03aafe1f56 100644
--- a/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.minnum.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.minnum.f16(half %a, half %b)
 declare <2 x half> @llvm.minnum.v2f16(<2 x half> %a, <2 x half> %b)
@@ -139,7 +139,7 @@ entry:
 ; SI:  v_min_f32_e32 v[[R_F32_1:[0-9]+]], 0x40400000, v[[A_F32_1]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
 ; VI-DAG:  v_mov_b32_e32 [[CONST3:v[0-9]+]], 0x4200
-; VI-DAG:  v_min_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[B_V2_F16]], [[CONST3]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-DAG:  v_min_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[A_V2_F16]], [[CONST3]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; VI-DAG:  v_min_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]]
 
 ; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
diff --git a/test/CodeGen/AMDGPU/llvm.rint.f16.ll b/test/CodeGen/AMDGPU/llvm.rint.f16.ll
index 92282083984bc..0e60f273129e8 100644
--- a/test/CodeGen/AMDGPU/llvm.rint.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.rint.f16.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=VI -check-prefix=GFX89 %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SIVI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=VI -check-prefix=GFX89 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 %s
 
 declare half @llvm.rint.f16(half %a)
 declare <2 x half> @llvm.rint.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.rint.f64.ll b/test/CodeGen/AMDGPU/llvm.rint.f64.ll
index 30ce8ed83ff18..e661c45d71870 100644
--- a/test/CodeGen/AMDGPU/llvm.rint.f64.ll
+++ b/test/CodeGen/AMDGPU/llvm.rint.f64.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}rint_f64:
 ; CI: v_rndne_f64_e32
diff --git a/test/CodeGen/AMDGPU/llvm.round.ll b/test/CodeGen/AMDGPU/llvm.round.ll
index 7e29147571f2f..48f1efb12fa4b 100644
--- a/test/CodeGen/AMDGPU/llvm.round.ll
+++ b/test/CodeGen/AMDGPU/llvm.round.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX89 -check-prefix=GFX9 -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
diff --git a/test/CodeGen/AMDGPU/llvm.sin.f16.ll b/test/CodeGen/AMDGPU/llvm.sin.f16.ll
index 08b9d9d873b49..839ff0636f974 100644
--- a/test/CodeGen/AMDGPU/llvm.sin.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.sin.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.sin.f16(half %a)
 declare <2 x half> @llvm.sin.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index 0e1358ecca226..48c82512466a8 100644
--- a/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.sqrt.f16(half %a)
 declare <2 x half> @llvm.sqrt.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/llvm.trunc.f16.ll b/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
index 37ee4e92c6379..a8ff7cbb0de3a 100644
--- a/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
+++ b/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 declare half @llvm.trunc.f16(half %a)
 declare <2 x half> @llvm.trunc.v2f16(<2 x half> %a)
diff --git a/test/CodeGen/AMDGPU/load-constant-i1.ll b/test/CodeGen/AMDGPU/load-constant-i1.ll
index c8abe5c77ee5e..789b5e9734d4a 100644
--- a/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}constant_load_i1:
 ; GCN: buffer_load_ubyte
diff --git a/test/CodeGen/AMDGPU/load-global-i1.ll b/test/CodeGen/AMDGPU/load-global-i1.ll
index cb3536a0c1284..b35922bac2318 100644
--- a/test/CodeGen/AMDGPU/load-global-i1.ll
+++ b/test/CodeGen/AMDGPU/load-global-i1.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}global_load_i1:
 ; GCN: buffer_load_ubyte
diff --git a/test/CodeGen/AMDGPU/load-global-i16.ll b/test/CodeGen/AMDGPU/load-global-i16.ll
index cb2495d5fdcf7..6d24334039512 100644
--- a/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -179,8 +179,8 @@ define amdgpu_kernel void @global_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)
 ; GCN-NOHSA: buffer_load_dwordx2
 ; GCN-HSA: flat_load_dwordx2
 
-; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}}
 ; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_LO:T[0-9]]], {{T[0-9]\.[XYZW]}}
+; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}}
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}},
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[ST_LO:T[0-9]]].XY, {{T[0-9]\.[XYZW]}},
 ; EGCM-DAG: VTX_READ_32 [[DST_LO:T[0-9]\.[XYZW]]], {{T[0-9]\.[XYZW]}}, 0, #1
@@ -188,8 +188,6 @@ define amdgpu_kernel void @global_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)
 ; TODO: This should use DST, but for some there are redundant MOVs
 ; EGCM: LSHR {{[* ]*}}[[ST_LO]].Y, {{T[0-9]\.[XYZW]}}, literal
 ; EGCM: 16
-; EGCM: AND_INT {{[* ]*}}[[ST_LO]].X, {{T[0-9]\.[XYZW]}}, literal
-; EGCM: AND_INT {{[* ]*}}[[ST_HI]].X, [[DST_HI]], literal
 define amdgpu_kernel void @global_zextload_v3i16_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i16> addrspace(1)* %in) {
 entry:
   %ld = load <3 x i16>, <3 x i16> addrspace(1)* %in
@@ -202,8 +200,8 @@ entry:
 ; GCN-NOHSA: buffer_load_dwordx2
 ; GCN-HSA: flat_load_dwordx2
 
-; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}}
 ; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_LO:T[0-9]]], {{T[0-9]\.[XYZW]}}
+; CM: MEM_RAT_CACHELESS STORE_DWORD [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}}
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[ST_HI:T[0-9]]].X, {{T[0-9]\.[XYZW]}},
 ; EG: MEM_RAT_CACHELESS STORE_RAW [[ST_LO:T[0-9]]].XY, {{T[0-9]\.[XYZW]}},
 ; EGCM-DAG: VTX_READ_32 [[DST_LO:T[0-9]\.[XYZW]]], {{T[0-9].[XYZW]}}, 0, #1
diff --git a/test/CodeGen/AMDGPU/load-global-i32.ll b/test/CodeGen/AMDGPU/load-global-i32.ll
index 6360d39666c77..246969303b04c 100644
--- a/test/CodeGen/AMDGPU/load-global-i32.ll
+++ b/test/CodeGen/AMDGPU/load-global-i32.ll
@@ -424,25 +424,25 @@ define amdgpu_kernel void @global_zextload_v16i32_to_v16i64(<16 x i64> addrspace
 ; GCN-NOHSA: buffer_store_dwordx4
 ; GCN-NOHSA: buffer_store_dwordx4
 
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
 
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
 
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
 
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
-; GCN-HSA: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
+; GCN-HSA-DAG: flat_store_dwordx4
 
 define amdgpu_kernel void @global_sextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(1)* %in) #0 {
   %ld = load <32 x i32>, <32 x i32> addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/load-global-i8.ll b/test/CodeGen/AMDGPU/load-global-i8.ll
index 3fe6bd26be14f..d7ebd46bc3e83 100644
--- a/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -352,22 +352,22 @@ define amdgpu_kernel void @global_zextload_v16i8_to_v16i32(<16 x i32> addrspace(
 
 ; EG: VTX_READ_128 [[DST:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1
 ; TODO: These should use DST, but for some there are redundant MOVs
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
-; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
+; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]*.[XYZW]}}, {{.*}}, 0.0, literal
 ; EG-DAG: 8
 ; EG-DAG: 8
 ; EG-DAG: 8
diff --git a/test/CodeGen/AMDGPU/load-hi16.ll b/test/CodeGen/AMDGPU/load-hi16.ll
new file mode 100644
index 0000000000000..8039ec372e458
--- /dev/null
+++ b/test/CodeGen/AMDGPU/load-hi16.ll
@@ -0,0 +1,657 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+
+; GCN-LABEL: {{^}}load_local_hi_v2i16_undeflo:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16_hi v0, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define <2 x i16> @load_local_hi_v2i16_undeflo(i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build = insertelement <2 x i16> undef, i16 %load, i32 1
+  ret <2 x i16> %build
+}
+
+; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define <2 x i16> @load_local_hi_v2i16_reglo(i16 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  ret <2 x i16> %build1
+}
+
+; Show that we get reasonable regalloc without physreg constraints.
+; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define void @load_local_hi_v2i16_reglo_vreg(i16 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define <2 x i16> @load_local_hi_v2i16_zerolo(i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 1
+  ret <2 x i16> %build
+}
+
+; FIXME: Remove m0 initialization
+; GCN-LABEL: {{^}}load_local_hi_v2i16_zerolo_shift:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16 v0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+; VI: v_lshlrev_b32_e32 v0, 16, v0
+define i32 @load_local_hi_v2i16_zerolo_shift(i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %zext = zext i16 %load to i32
+  %shift = shl i32 %zext, 16
+  ret i32 %shift
+}
+
+; GCN-LABEL: {{^}}load_local_hi_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define void @load_local_hi_v2f16_reglo_vreg(half addrspace(3)* %in, half %reg) #0 {
+entry:
+  %load = load half, half addrspace(3)* %in
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u8_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u8 v
+define void @load_local_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_hi_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_i8_d16_hi v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_i8 v
+define void @load_local_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_hi_v2i16_reglo_vreg(i16 addrspace(1)* %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 -2047
+  %load = load i16, i16 addrspace(1)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_hi_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_hi_v2f16_reglo_vreg(half addrspace(1)* %in, half %reg) #0 {
+entry:
+  %gep = getelementptr inbounds half, half addrspace(1)* %in, i64 -2047
+  %load = load half, half addrspace(1)* %gep
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_ubyte_d16_hi v2, v[0:1], off offset:-4095
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
+  %load = load i8, i8 addrspace(1)* %gep
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_hi_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_sbyte_d16_hi v2, v[0:1], off offset:-4095
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
+  %load = load i8, i8 addrspace(1)* %gep
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: load_flat_hi_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort v{{[0-9]+}}
+; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; VI: v_or_b32_sdwa
+define void @load_flat_hi_v2i16_reglo_vreg(i16 addrspace(4)* %in, i16 %reg) #0 {
+entry:
+  %load = load i16, i16 addrspace(4)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_hi_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_short_d16_hi v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort v{{[0-9]+}}
+; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; VI: v_or_b32_sdwa
+define void @load_flat_hi_v2f16_reglo_vreg(half addrspace(4)* %in, half %reg) #0 {
+entry:
+  %load = load half, half addrspace(4)* %in
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_ubyte_d16_hi v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ubyte v{{[0-9]+}}
+; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; VI: v_or_b32_sdwa
+define void @load_flat_hi_v2i16_reglo_vreg_zexti8(i8 addrspace(4)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(4)* %in
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_hi_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_sbyte_d16_hi v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_sbyte v{{[0-9]+}}
+; VI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16,
+; VI: v_or_b32_sdwa
+define void @load_flat_hi_v2i16_reglo_vreg_sexti8(i8 addrspace(4)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(4)* %in
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+define void @load_private_hi_v2i16_reglo_vreg(i16* byval %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16* %in, i64 2045
+  %load = load i16, i16* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+define void @load_private_hi_v2f16_reglo_vreg(half* byval %in, half %reg) #0 {
+entry:
+  %gep = getelementptr inbounds half, half* %in, i64 2045
+  %load = load half, half* %gep
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff:
+; GCN: s_waitcnt
+; GFX9: buffer_load_short_d16_hi v0, off, s[0:3], s4 offset:4094{{$}}
+; GFX9: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_hi_v2i16_reglo_vreg_nooff(i16* byval %in, i16 %reg) #0 {
+entry:
+  %load = load volatile i16, i16* inttoptr (i32 4094 to i16*)
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_short_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_hi_v2f16_reglo_vreg_nooff(half* %in, half %reg) #0 {
+entry:
+  %load = load volatile half, half* inttoptr (i32 4094 to half*)
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9: buffer_load_ubyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+define void @load_private_hi_v2i16_reglo_vreg_zexti8(i8* byval %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i8, i8* %in, i64 4091
+  %load = load i8, i8* %gep
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9: buffer_load_sbyte_d16_hi v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+define void @load_private_hi_v2i16_reglo_vreg_sexti8(i8* byval %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i8, i8* %in, i64 4091
+  %load = load i8, i8* %gep
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_hi_v2i16_reglo_vreg_nooff_zexti8(i8* %in, i16 %reg) #0 {
+entry:
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_nooff_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_sbyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_hi_v2i16_reglo_vreg_nooff_sexti8(i8* %in, i16 %reg) #0 {
+entry:
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2f16_reglo_vreg_nooff_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_ubyte_d16_hi v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_hi_v2f16_reglo_vreg_nooff_zexti8(i8* %in, half %reg) #0 {
+entry:
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = zext i8 %load to i16
+  %bc.ext = bitcast i16 %ext to half
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %bc.ext, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_constant_hi_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort
+define void @load_constant_hi_v2i16_reglo_vreg(i16 addrspace(2)* %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(2)* %in, i64 -2047
+  %load = load i16, i16 addrspace(2)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: load_constant_hi_v2f16_reglo_vreg
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort
+define void @load_constant_hi_v2f16_reglo_vreg(half addrspace(2)* %in, half %reg) #0 {
+entry:
+  %gep = getelementptr inbounds half, half addrspace(2)* %in, i64 -2047
+  %load = load half, half addrspace(2)* %gep
+  %build0 = insertelement <2 x half> undef, half %reg, i32 0
+  %build1 = insertelement <2 x half> %build0, half %load, i32 1
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; Local object gives known offset, so requires converting from offen
+; to offset variant.
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_short_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4094
+define void @load_private_hi_v2i16_reglo_vreg_to_offset(i16 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i16], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i16], [4096 x i16]* %obj1, i32 0, i32 2025
+  %load = load i16, i16* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_sbyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
+define void @load_private_hi_v2i16_reglo_vreg_sexti8_to_offset(i16 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load i8, i8* %gep
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_hi_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_ubyte_d16_hi v{{[0-9]+}}, off, s[0:3], s5 offset:4095
+define void @load_private_hi_v2i16_reglo_vreg_zexti8_to_offset(i16 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load i8, i8* %gep
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 1
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; FIXME: Remove m0 init and waitcnt between reads
+; FIXME: Is there a cost to using the extload over not?
+; GCN-LABEL: {{^}}load_local_v2i16_split:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16_hi v1, v0 offset:2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: s_setpc_b64
+define <2 x i16> @load_local_v2i16_split(i16 addrspace(3)* %in) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %in, i32 1
+  %load0 = load volatile i16, i16 addrspace(3)* %in
+  %load1 = load volatile i16, i16 addrspace(3)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
+  ret <2 x i16> %build1
+}
+
+; FIXME: Remove waitcnt between reads
+; GCN-LABEL: {{^}}load_global_v2i16_split:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_ushort v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64
+define <2 x i16> @load_global_v2i16_split(i16 addrspace(1)* %in) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 1
+  %load0 = load volatile i16, i16 addrspace(1)* %in
+  %load1 = load volatile i16, i16 addrspace(1)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
+  ret <2 x i16> %build1
+}
+
+; FIXME: Remove waitcnt between reads
+; GCN-LABEL: {{^}}load_flat_v2i16_split:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_ushort v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: flat_load_short_d16_hi v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64
+define <2 x i16> @load_flat_v2i16_split(i16 addrspace(4)* %in) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(4)* %in, i64 1
+  %load0 = load volatile i16, i16 addrspace(4)* %in
+  %load1 = load volatile i16, i16 addrspace(4)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
+  ret <2 x i16> %build1
+}
+
+; FIXME: Remove waitcnt between reads
+; GCN-LABEL: {{^}}load_constant_v2i16_split:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_ushort v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_load_short_d16_hi v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v2
+; GFX9-NEXT: s_setpc_b64
+define <2 x i16> @load_constant_v2i16_split(i16 addrspace(2)* %in) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16 addrspace(2)* %in, i64 1
+  %load0 = load volatile i16, i16 addrspace(2)* %in
+  %load1 = load volatile i16, i16 addrspace(2)* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
+  ret <2 x i16> %build1
+}
+
+; FIXME: Remove m0 init and waitcnt between reads
+; FIXME: Is there a cost to using the extload over not?
+; GCN-LABEL: {{^}}load_private_v2i16_split:
+; GCN: s_waitcnt
+; GFX9: buffer_load_ushort v0, off, s[0:3], s5 offset:4{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: buffer_load_short_d16_hi v0, off, s[0:3], s5 offset:6
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define <2 x i16> @load_private_v2i16_split(i16* byval %in) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16* %in, i32 1
+  %load0 = load volatile i16, i16* %in
+  %load1 = load volatile i16, i16* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %load0, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load1, i32 1
+  ret <2 x i16> %build1
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/load-lo16.ll b/test/CodeGen/AMDGPU/load-lo16.ll
new file mode 100644
index 0000000000000..98a790ddd6ff5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/load-lo16.ll
@@ -0,0 +1,650 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_undeflo:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16 v0, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16
+define <2 x i16> @load_local_lo_v2i16_undeflo(i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build = insertelement <2 x i16> undef, i16 %load, i32 0
+  ret <2 x i16> %build
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16 v0, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16
+define <2 x i16> @load_local_lo_v2i16_reglo(i16 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
+  ret <2 x i16> %build1
+}
+
+; Show that we get reasonable regalloc without physreg constraints.
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16 v0, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v0, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16
+define void @load_local_lo_v2i16_reglo_vreg(i16 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 0
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_zerolo:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: ds_read_u16_d16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define <2 x i16> @load_local_lo_v2i16_zerolo(i16 addrspace(3)* %in) #0 {
+entry:
+  %load = load i16, i16 addrspace(3)* %in
+  %build = insertelement <2 x i16> zeroinitializer, i16 %load, i32 0
+  ret <2 x i16> %build
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2f16_fpimm:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX9-NEXT: ds_read_u16_d16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define <2 x half> @load_local_lo_v2f16_fpimm(half addrspace(3)* %in) #0 {
+entry:
+  %load = load half, half addrspace(3)* %in
+  %build = insertelement <2 x half> <half 0.0, half 2.0>, half %load, i32 0
+  ret <2 x half> %build
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2f16_reghi_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u16_d16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u16 v
+define void @load_local_lo_v2f16_reghi_vreg(half addrspace(3)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %load = load half, half addrspace(3)* %in
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2f16_reglo_vreg:
+
+; GFX9: ds_read_u16 v
+; GFX9: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
+; GFX9: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, v{{[0-9]+}}
+; GFX9: global_store_dword
+
+; VI: ds_read_u16 v
+define void @load_local_lo_v2f16_reglo_vreg(half addrspace(3)* %in, half %reg) #0 {
+entry:
+  %load = load half, half addrspace(3)* %in
+  %build0 = insertelement <2 x half> undef, half %reg, i32 1
+  %build1 = insertelement <2 x half> %build0, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_u8_d16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u8 v
+define void @load_local_lo_v2i16_reghi_vreg_zexti8(i8 addrspace(3)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9: ds_read_u8 v
+; GFX9: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_u8 v
+define void @load_local_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = zext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_read_i8_d16 v1, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v1, off{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: ds_read_i8 v
+define void @load_local_lo_v2i16_reghi_vreg_sexti8(i8 addrspace(3)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_local_lo_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9: ds_read_i8 v
+; GFX9: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
+; GFX9: v_lshl_or_b32 v{{[0-9]+}}, v{{[0-9]+}}, 16, v{{[0-9]+}}
+
+; VI: ds_read_i8 v
+define void @load_local_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(3)* %in, i16 %reg) #0 {
+entry:
+  %load = load i8, i8 addrspace(3)* %in
+  %ext = sext i8 %load to i16
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build1 = insertelement <2 x i16> %build0, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_lo_v2i16_reglo_vreg(i16 addrspace(1)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i64 -2047
+  %load = load i16, i16 addrspace(1)* %gep
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_lo_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_lo_v2f16_reglo_vreg(half addrspace(1)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %gep = getelementptr inbounds half, half addrspace(1)* %in, i64 -2047
+  %load = load half, half addrspace(1)* %gep
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_ubyte_d16 v2, v[0:1], off offset:-4095
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
+  %load = load i8, i8 addrspace(1)* %gep
+  %ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_global_lo_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_sbyte_d16 v2, v[0:1], off offset:-4095
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+define void @load_global_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(1)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 -4095
+  %load = load i8, i8 addrspace(1)* %gep
+  %ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_lo_v2i16_reghi_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_short_d16 v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort v{{[0-9]+}}
+; VI: v_or_b32_e32
+define void @load_flat_lo_v2i16_reghi_vreg(i16 addrspace(4)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load i16, i16 addrspace(4)* %in
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_lo_v2f16_reghi_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_short_d16 v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort v{{[0-9]+}}
+; VI: v_or_b32_e32
+define void @load_flat_lo_v2f16_reghi_vreg(half addrspace(4)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %load = load half, half addrspace(4)* %in
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_lo_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_ubyte_d16 v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ubyte v{{[0-9]+}}
+; VI: v_or_b32_e32
+define void @load_flat_lo_v2i16_reglo_vreg_zexti8(i8 addrspace(4)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load i8, i8 addrspace(4)* %in
+  %ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_flat_lo_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_load_sbyte_d16 v2, v[0:1]
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v[0:1], v2
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_sbyte v{{[0-9]+}}
+; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+
+define void @load_flat_lo_v2i16_reglo_vreg_sexti8(i8 addrspace(4)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load i8, i8 addrspace(4)* %in
+  %ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9: buffer_load_short_d16 v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+define void @load_private_lo_v2i16_reglo_vreg(i16* byval %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i16, i16* %in, i64 2045
+  %load = load i16, i16* %gep
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reghi_vreg:
+; GCN: s_waitcnt
+; GFX9: buffer_load_ushort v1, off, s[0:3], s5 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9: v_and_b32
+; GFX9: v_lshl_or_b32
+
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+define void @load_private_lo_v2i16_reghi_vreg(i16* byval %in, i16 %reg) #0 {
+entry:
+  %gep = getelementptr inbounds i16, i16* %in, i64 2045
+  %load = load i16, i16* %gep
+  %build0 = insertelement <2 x i16> undef, i16 %reg, i32 1
+  %build1 = insertelement <2 x i16> %build0, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9: buffer_load_short_d16 v0, off, s[0:3], s5 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s5 offset:4094{{$}}
+define void @load_private_lo_v2f16_reglo_vreg(half* byval %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %gep = getelementptr inbounds half, half* %in, i64 2045
+  %load = load half, half* %gep
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_short_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2i16_reglo_vreg_nooff(i16* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load volatile i16, i16* inttoptr (i32 4094 to i16*)
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reghi_vreg_nooff:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_short_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2i16_reghi_vreg_nooff(i16* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load volatile i16, i16* inttoptr (i32 4094 to i16*)
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_nooff:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_short_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ushort v{{[0-9]+}}, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2f16_reglo_vreg_nooff(half* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %load = load volatile half, half* inttoptr (i32 4094 to half*)
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_zexti8:
+; GCN: s_waitcnt
+; GFX9: buffer_load_ubyte_d16 v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+define void @load_private_lo_v2i16_reglo_vreg_zexti8(i8* byval %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i8, i8* %in, i64 4091
+  %load = load i8, i8* %gep
+  %ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_sexti8:
+; GCN: s_waitcnt
+; GFX9: buffer_load_sbyte_d16 v0, off, s[0:3], s5 offset:4095{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v0
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_sbyte v{{[0-9]+}}, off, s[0:3], s5 offset:4095{{$}}
+define void @load_private_lo_v2i16_reglo_vreg_sexti8(i8* byval %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i8, i8* %in, i64 4091
+  %load = load i8, i8* %gep
+  %ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_ubyte_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2i16_reglo_vreg_nooff_zexti8(i8* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_nooff_sexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_sbyte_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_sbyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2i16_reglo_vreg_nooff_sexti8(i8* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2f16_reglo_vreg_nooff_zexti8:
+; GCN: s_waitcnt
+; GFX9-NEXT: buffer_load_ubyte_d16 v1, off, s[0:3], s4 offset:4094{{$}}
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v1
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: buffer_load_ubyte v0, off, s[0:3], s4 offset:4094{{$}}
+define void @load_private_lo_v2f16_reglo_vreg_nooff_zexti8(i8* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %load = load volatile i8, i8* inttoptr (i32 4094 to i8*)
+  %ext = zext i8 %load to i16
+  %bc.ext = bitcast i16 %ext to half
+  %build1 = insertelement <2 x half> %reg.bc, half %bc.ext, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_constant_lo_v2i16_reglo_vreg:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort
+define void @load_constant_lo_v2i16_reglo_vreg(i16 addrspace(2)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %gep = getelementptr inbounds i16, i16 addrspace(2)* %in, i64 -2047
+  %load = load i16, i16 addrspace(2)* %gep
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: load_constant_lo_v2f16_reglo_vreg
+; GCN: s_waitcnt
+; GFX9-NEXT: global_load_short_d16 v2, v[0:1], off offset:-4094
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: global_store_dword
+; GFX9-NEXT: s_waitcnt
+; GFX9-NEXT: s_setpc_b64
+
+; VI: flat_load_ushort
+define void @load_constant_lo_v2f16_reglo_vreg(half addrspace(2)* %in, i32 %reg) #0 {
+entry:
+  %reg.bc = bitcast i32 %reg to <2 x half>
+  %gep = getelementptr inbounds half, half addrspace(2)* %in, i64 -2047
+  %load = load half, half addrspace(2)* %gep
+  %build1 = insertelement <2 x half> %reg.bc, half %load, i32 0
+  store <2 x half> %build1, <2 x half> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_short_d16 v0, off, s[0:3], s5 offset:4094
+
+; VI: buffer_load_ushort v
+define void @load_private_lo_v2i16_reglo_vreg_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i16], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i16], [4096 x i16]* %obj1, i32 0, i32 2025
+  %load = load volatile i16, i16* %gep
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_sexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_sbyte_d16 v0, off, s[0:3], s5 offset:4095
+
+; VI: buffer_load_sbyte v
+define void @load_private_lo_v2i16_reglo_vreg_sexti8_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load volatile i8, i8* %gep
+  %load.ext = sext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load.ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_lo_v2i16_reglo_vreg_zexti8_to_offset:
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_load_ubyte_d16 v0, off, s[0:3], s5 offset:4095
+
+; VI: buffer_load_ubyte v
+define void @load_private_lo_v2i16_reglo_vreg_zexti8_to_offset(i32 %reg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %reg.bc = bitcast i32 %reg to <2 x i16>
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %load = load volatile i8, i8* %gep
+  %load.ext = zext i8 %load to i16
+  %build1 = insertelement <2 x i16> %reg.bc, i16 %load.ext, i32 0
+  store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/load-local-f32.ll b/test/CodeGen/AMDGPU/load-local-f32.ll
index 09d7145424de8..f035d22018a71 100644
--- a/test/CodeGen/AMDGPU/load-local-f32.ll
+++ b/test/CodeGen/AMDGPU/load-local-f32.ll
@@ -1,9 +1,10 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}load_f32_local:
-; GCN: s_mov_b32 m0
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -15,7 +16,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}load_v2f32_local:
-; GCN: s_mov_b32 m0
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_b64
 
 ; EG: LDS_READ_RET
@@ -29,6 +32,9 @@ entry:
 
 ; FIXME: should this do a read2_b64?
 ; FUNC-LABEL: {{^}}local_load_v3f32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:8
 ; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+$}}
 ; GCN: s_waitcnt
@@ -46,6 +52,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4f32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 
 ; EG: LDS_READ_RET
@@ -60,6 +69,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8f32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 
@@ -79,6 +91,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16f32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
diff --git a/test/CodeGen/AMDGPU/load-local-f64.ll b/test/CodeGen/AMDGPU/load-local-f64.ll
index 9ad6c087bf2ef..ffb67101fd786 100644
--- a/test/CodeGen/AMDGPU/load-local-f64.ll
+++ b/test/CodeGen/AMDGPU/load-local-f64.ll
@@ -1,9 +1,13 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_b64 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
 ; GCN: ds_write_b64 v{{[0-9]+}}, [[VAL]]
 
@@ -16,6 +20,9 @@ define amdgpu_kernel void @local_load_f64(double addrspace(3)* %out, double addr
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 
 ; EG: LDS_READ_RET
@@ -30,6 +37,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read2_b64
 ; GCN-DAG: ds_read_b64
 
@@ -47,6 +57,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 
@@ -67,6 +80,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
@@ -96,6 +112,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16f64:
+; SICIV: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
diff --git a/test/CodeGen/AMDGPU/load-local-i1.ll b/test/CodeGen/AMDGPU/load-local-i1.ll
index e8f134b1fb2ef..0320debc828d3 100644
--- a/test/CodeGen/AMDGPU/load-local-i1.ll
+++ b/test/CodeGen/AMDGPU/load-local-i1.ll
@@ -1,8 +1,12 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_u8
 ; GCN: v_and_b32_e32 v{{[0-9]+}}, 1
 ; GCN: ds_write_b8
@@ -17,6 +21,8 @@ define amdgpu_kernel void @local_load_i1(i1 addrspace(3)* %out, i1 addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v2i1(<2 x i1> addrspace(3)* %out, <2 x i1> addrspace(3)* %in) #0 {
   %load = load <2 x i1>, <2 x i1> addrspace(3)* %in
   store <2 x i1> %load, <2 x i1> addrspace(3)* %out
@@ -24,6 +30,8 @@ define amdgpu_kernel void @local_load_v2i1(<2 x i1> addrspace(3)* %out, <2 x i1>
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v3i1(<3 x i1> addrspace(3)* %out, <3 x i1> addrspace(3)* %in) #0 {
   %load = load <3 x i1>, <3 x i1> addrspace(3)* %in
   store <3 x i1> %load, <3 x i1> addrspace(3)* %out
@@ -31,6 +39,8 @@ define amdgpu_kernel void @local_load_v3i1(<3 x i1> addrspace(3)* %out, <3 x i1>
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v4i1(<4 x i1> addrspace(3)* %out, <4 x i1> addrspace(3)* %in) #0 {
   %load = load <4 x i1>, <4 x i1> addrspace(3)* %in
   store <4 x i1> %load, <4 x i1> addrspace(3)* %out
@@ -38,6 +48,8 @@ define amdgpu_kernel void @local_load_v4i1(<4 x i1> addrspace(3)* %out, <4 x i1>
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v8i1(<8 x i1> addrspace(3)* %out, <8 x i1> addrspace(3)* %in) #0 {
   %load = load <8 x i1>, <8 x i1> addrspace(3)* %in
   store <8 x i1> %load, <8 x i1> addrspace(3)* %out
@@ -45,6 +57,8 @@ define amdgpu_kernel void @local_load_v8i1(<8 x i1> addrspace(3)* %out, <8 x i1>
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v16i1(<16 x i1> addrspace(3)* %out, <16 x i1> addrspace(3)* %in) #0 {
   %load = load <16 x i1>, <16 x i1> addrspace(3)* %in
   store <16 x i1> %load, <16 x i1> addrspace(3)* %out
@@ -52,6 +66,8 @@ define amdgpu_kernel void @local_load_v16i1(<16 x i1> addrspace(3)* %out, <16 x
 }
 
 ; FUNC-LABEL: {{^}}local_load_v32i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v32i1(<32 x i1> addrspace(3)* %out, <32 x i1> addrspace(3)* %in) #0 {
   %load = load <32 x i1>, <32 x i1> addrspace(3)* %in
   store <32 x i1> %load, <32 x i1> addrspace(3)* %out
@@ -59,6 +75,8 @@ define amdgpu_kernel void @local_load_v32i1(<32 x i1> addrspace(3)* %out, <32 x
 }
 
 ; FUNC-LABEL: {{^}}local_load_v64i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_load_v64i1(<64 x i1> addrspace(3)* %out, <64 x i1> addrspace(3)* %in) #0 {
   %load = load <64 x i1>, <64 x i1> addrspace(3)* %in
   store <64 x i1> %load, <64 x i1> addrspace(3)* %out
@@ -66,6 +84,9 @@ define amdgpu_kernel void @local_load_v64i1(<64 x i1> addrspace(3)* %out, <64 x
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i1_to_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_u8
 ; GCN: ds_write_b32
 define amdgpu_kernel void @local_zextload_i1_to_i32(i32 addrspace(3)* %out, i1 addrspace(3)* %in) #0 {
@@ -76,6 +97,9 @@ define amdgpu_kernel void @local_zextload_i1_to_i32(i32 addrspace(3)* %out, i1 a
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i1_to_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_u8
 ; GCN: v_bfe_i32 {{v[0-9]+}}, {{v[0-9]+}}, 0, 1{{$}}
 ; GCN: ds_write_b32
@@ -90,6 +114,8 @@ define amdgpu_kernel void @local_sextload_i1_to_i32(i32 addrspace(3)* %out, i1 a
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i1_to_v1i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v1i1_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i1> addrspace(3)* %in) #0 {
   %load = load <1 x i1>, <1 x i1> addrspace(3)* %in
   %ext = zext <1 x i1> %load to <1 x i32>
@@ -98,6 +124,8 @@ define amdgpu_kernel void @local_zextload_v1i1_to_v1i32(<1 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i1_to_v1i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v1i1_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i1> addrspace(3)* %in) #0 {
   %load = load <1 x i1>, <1 x i1> addrspace(3)* %in
   %ext = sext <1 x i1> %load to <1 x i32>
@@ -106,6 +134,8 @@ define amdgpu_kernel void @local_sextload_v1i1_to_v1i32(<1 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i1_to_v2i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v2i1_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i1> addrspace(3)* %in) #0 {
   %load = load <2 x i1>, <2 x i1> addrspace(3)* %in
   %ext = zext <2 x i1> %load to <2 x i32>
@@ -114,6 +144,8 @@ define amdgpu_kernel void @local_zextload_v2i1_to_v2i32(<2 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i1_to_v2i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v2i1_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i1> addrspace(3)* %in) #0 {
   %load = load <2 x i1>, <2 x i1> addrspace(3)* %in
   %ext = sext <2 x i1> %load to <2 x i32>
@@ -122,6 +154,8 @@ define amdgpu_kernel void @local_sextload_v2i1_to_v2i32(<2 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v3i1_to_v3i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v3i1_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i1> addrspace(3)* %in) #0 {
   %load = load <3 x i1>, <3 x i1> addrspace(3)* %in
   %ext = zext <3 x i1> %load to <3 x i32>
@@ -130,6 +164,8 @@ define amdgpu_kernel void @local_zextload_v3i1_to_v3i32(<3 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v3i1_to_v3i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v3i1_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i1> addrspace(3)* %in) #0 {
   %load = load <3 x i1>, <3 x i1> addrspace(3)* %in
   %ext = sext <3 x i1> %load to <3 x i32>
@@ -138,6 +174,8 @@ define amdgpu_kernel void @local_sextload_v3i1_to_v3i32(<3 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i1_to_v4i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v4i1_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i1> addrspace(3)* %in) #0 {
   %load = load <4 x i1>, <4 x i1> addrspace(3)* %in
   %ext = zext <4 x i1> %load to <4 x i32>
@@ -146,6 +184,8 @@ define amdgpu_kernel void @local_zextload_v4i1_to_v4i32(<4 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i1_to_v4i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v4i1_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i1> addrspace(3)* %in) #0 {
   %load = load <4 x i1>, <4 x i1> addrspace(3)* %in
   %ext = sext <4 x i1> %load to <4 x i32>
@@ -154,6 +194,8 @@ define amdgpu_kernel void @local_sextload_v4i1_to_v4i32(<4 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i1_to_v8i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v8i1_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i1> addrspace(3)* %in) #0 {
   %load = load <8 x i1>, <8 x i1> addrspace(3)* %in
   %ext = zext <8 x i1> %load to <8 x i32>
@@ -162,6 +204,8 @@ define amdgpu_kernel void @local_zextload_v8i1_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i1_to_v8i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v8i1_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i1> addrspace(3)* %in) #0 {
   %load = load <8 x i1>, <8 x i1> addrspace(3)* %in
   %ext = sext <8 x i1> %load to <8 x i32>
@@ -170,6 +214,8 @@ define amdgpu_kernel void @local_sextload_v8i1_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i1_to_v16i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v16i1_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i1> addrspace(3)* %in) #0 {
   %load = load <16 x i1>, <16 x i1> addrspace(3)* %in
   %ext = zext <16 x i1> %load to <16 x i32>
@@ -178,6 +224,8 @@ define amdgpu_kernel void @local_zextload_v16i1_to_v16i32(<16 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i1_to_v16i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v16i1_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i1> addrspace(3)* %in) #0 {
   %load = load <16 x i1>, <16 x i1> addrspace(3)* %in
   %ext = sext <16 x i1> %load to <16 x i32>
@@ -186,6 +234,8 @@ define amdgpu_kernel void @local_sextload_v16i1_to_v16i32(<16 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i1_to_v32i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v32i1_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i1> addrspace(3)* %in) #0 {
   %load = load <32 x i1>, <32 x i1> addrspace(3)* %in
   %ext = zext <32 x i1> %load to <32 x i32>
@@ -194,6 +244,8 @@ define amdgpu_kernel void @local_zextload_v32i1_to_v32i32(<32 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i1_to_v32i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v32i1_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i1> addrspace(3)* %in) #0 {
   %load = load <32 x i1>, <32 x i1> addrspace(3)* %in
   %ext = sext <32 x i1> %load to <32 x i32>
@@ -202,6 +254,8 @@ define amdgpu_kernel void @local_sextload_v32i1_to_v32i32(<32 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v64i1_to_v64i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v64i1_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i1> addrspace(3)* %in) #0 {
   %load = load <64 x i1>, <64 x i1> addrspace(3)* %in
   %ext = zext <64 x i1> %load to <64 x i32>
@@ -210,6 +264,8 @@ define amdgpu_kernel void @local_zextload_v64i1_to_v64i32(<64 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v64i1_to_v64i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v64i1_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i1> addrspace(3)* %in) #0 {
   %load = load <64 x i1>, <64 x i1> addrspace(3)* %in
   %ext = sext <64 x i1> %load to <64 x i32>
@@ -218,6 +274,9 @@ define amdgpu_kernel void @local_sextload_v64i1_to_v64i32(<64 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i1_to_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read_u8 [[LOAD:v[0-9]+]],
 ; GCN-DAG: v_mov_b32_e32 {{v[0-9]+}}, 0{{$}}
 ; GCN: ds_write_b64
@@ -229,6 +288,9 @@ define amdgpu_kernel void @local_zextload_i1_to_i64(i64 addrspace(3)* %out, i1 a
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i1_to_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_u8 [[LOAD:v[0-9]+]],
 ; GCN: v_bfe_i32 [[BFE:v[0-9]+]], {{v[0-9]+}}, 0, 1{{$}}
 ; GCN: v_ashrrev_i32_e32 v{{[0-9]+}}, 31, [[BFE]]
@@ -241,6 +303,8 @@ define amdgpu_kernel void @local_sextload_i1_to_i64(i64 addrspace(3)* %out, i1 a
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i1_to_v1i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v1i1_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i1> addrspace(3)* %in) #0 {
   %load = load <1 x i1>, <1 x i1> addrspace(3)* %in
   %ext = zext <1 x i1> %load to <1 x i64>
@@ -249,6 +313,8 @@ define amdgpu_kernel void @local_zextload_v1i1_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i1_to_v1i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v1i1_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i1> addrspace(3)* %in) #0 {
   %load = load <1 x i1>, <1 x i1> addrspace(3)* %in
   %ext = sext <1 x i1> %load to <1 x i64>
@@ -257,6 +323,8 @@ define amdgpu_kernel void @local_sextload_v1i1_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i1_to_v2i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v2i1_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i1> addrspace(3)* %in) #0 {
   %load = load <2 x i1>, <2 x i1> addrspace(3)* %in
   %ext = zext <2 x i1> %load to <2 x i64>
@@ -265,6 +333,8 @@ define amdgpu_kernel void @local_zextload_v2i1_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i1_to_v2i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v2i1_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i1> addrspace(3)* %in) #0 {
   %load = load <2 x i1>, <2 x i1> addrspace(3)* %in
   %ext = sext <2 x i1> %load to <2 x i64>
@@ -273,6 +343,8 @@ define amdgpu_kernel void @local_sextload_v2i1_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v3i1_to_v3i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v3i1_to_v3i64(<3 x i64> addrspace(3)* %out, <3 x i1> addrspace(3)* %in) #0 {
   %load = load <3 x i1>, <3 x i1> addrspace(3)* %in
   %ext = zext <3 x i1> %load to <3 x i64>
@@ -281,6 +353,8 @@ define amdgpu_kernel void @local_zextload_v3i1_to_v3i64(<3 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v3i1_to_v3i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v3i1_to_v3i64(<3 x i64> addrspace(3)* %out, <3 x i1> addrspace(3)* %in) #0 {
   %load = load <3 x i1>, <3 x i1> addrspace(3)* %in
   %ext = sext <3 x i1> %load to <3 x i64>
@@ -289,6 +363,8 @@ define amdgpu_kernel void @local_sextload_v3i1_to_v3i64(<3 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i1_to_v4i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v4i1_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i1> addrspace(3)* %in) #0 {
   %load = load <4 x i1>, <4 x i1> addrspace(3)* %in
   %ext = zext <4 x i1> %load to <4 x i64>
@@ -297,6 +373,8 @@ define amdgpu_kernel void @local_zextload_v4i1_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i1_to_v4i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v4i1_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i1> addrspace(3)* %in) #0 {
   %load = load <4 x i1>, <4 x i1> addrspace(3)* %in
   %ext = sext <4 x i1> %load to <4 x i64>
@@ -305,6 +383,8 @@ define amdgpu_kernel void @local_sextload_v4i1_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i1_to_v8i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v8i1_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i1> addrspace(3)* %in) #0 {
   %load = load <8 x i1>, <8 x i1> addrspace(3)* %in
   %ext = zext <8 x i1> %load to <8 x i64>
@@ -313,6 +393,8 @@ define amdgpu_kernel void @local_zextload_v8i1_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i1_to_v8i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v8i1_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i1> addrspace(3)* %in) #0 {
   %load = load <8 x i1>, <8 x i1> addrspace(3)* %in
   %ext = sext <8 x i1> %load to <8 x i64>
@@ -321,6 +403,8 @@ define amdgpu_kernel void @local_sextload_v8i1_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i1_to_v16i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v16i1_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i1> addrspace(3)* %in) #0 {
   %load = load <16 x i1>, <16 x i1> addrspace(3)* %in
   %ext = zext <16 x i1> %load to <16 x i64>
@@ -329,6 +413,8 @@ define amdgpu_kernel void @local_zextload_v16i1_to_v16i64(<16 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i1_to_v16i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v16i1_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i1> addrspace(3)* %in) #0 {
   %load = load <16 x i1>, <16 x i1> addrspace(3)* %in
   %ext = sext <16 x i1> %load to <16 x i64>
@@ -337,6 +423,8 @@ define amdgpu_kernel void @local_sextload_v16i1_to_v16i64(<16 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i1_to_v32i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v32i1_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i1> addrspace(3)* %in) #0 {
   %load = load <32 x i1>, <32 x i1> addrspace(3)* %in
   %ext = zext <32 x i1> %load to <32 x i64>
@@ -345,6 +433,8 @@ define amdgpu_kernel void @local_zextload_v32i1_to_v32i64(<32 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i1_to_v32i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v32i1_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i1> addrspace(3)* %in) #0 {
   %load = load <32 x i1>, <32 x i1> addrspace(3)* %in
   %ext = sext <32 x i1> %load to <32 x i64>
@@ -353,6 +443,8 @@ define amdgpu_kernel void @local_sextload_v32i1_to_v32i64(<32 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v64i1_to_v64i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_zextload_v64i1_to_v64i64(<64 x i64> addrspace(3)* %out, <64 x i1> addrspace(3)* %in) #0 {
   %load = load <64 x i1>, <64 x i1> addrspace(3)* %in
   %ext = zext <64 x i1> %load to <64 x i64>
@@ -361,6 +453,8 @@ define amdgpu_kernel void @local_zextload_v64i1_to_v64i64(<64 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v64i1_to_v64i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 define amdgpu_kernel void @local_sextload_v64i1_to_v64i64(<64 x i64> addrspace(3)* %out, <64 x i1> addrspace(3)* %in) #0 {
   %load = load <64 x i1>, <64 x i1> addrspace(3)* %in
   %ext = sext <64 x i1> %load to <64 x i64>
diff --git a/test/CodeGen/AMDGPU/load-local-i16.ll b/test/CodeGen/AMDGPU/load-local-i16.ll
index 7de3f3b28c6dd..d3557c14540ce 100644
--- a/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -1,8 +1,12 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,GFX89,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GFX89,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_u16 v{{[0-9]+}}
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
@@ -18,6 +22,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b32
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
@@ -33,6 +40,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 ; GCN-DAG: ds_write_b32
 ; GCN-DAG: ds_write_b16
@@ -47,6 +57,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 
 ; EG: LDS_READ_RET
@@ -59,6 +72,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 
 ; EG: LDS_READ_RET
@@ -73,6 +89,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i16:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:3{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}}
 
@@ -94,6 +113,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i16_to_i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_u16
 ; GCN: ds_write_b32
 
@@ -111,7 +133,10 @@ define amdgpu_kernel void @local_zextload_i16_to_i32(i32 addrspace(3)* %out, i16
 
 ; FUNC-LABEL: {{^}}local_sextload_i16_to_i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_i16
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
@@ -129,6 +154,9 @@ define amdgpu_kernel void @local_sextload_i16_to_i32(i32 addrspace(3)* %out, i16
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i16_to_v1i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_u16
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
@@ -144,6 +172,9 @@ define amdgpu_kernel void @local_zextload_v1i16_to_v1i32(<1 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i16_to_v1i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_i16
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
@@ -162,7 +193,9 @@ define amdgpu_kernel void @local_sextload_v1i16_to_v1i32(<1 x i32> addrspace(3)*
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i16_to_v2i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -175,7 +208,9 @@ define amdgpu_kernel void @local_zextload_v2i16_to_v2i32(<2 x i32> addrspace(3)*
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i16_to_v2i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -189,6 +224,9 @@ define amdgpu_kernel void @local_sextload_v2i16_to_v2i32(<2 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_local_zextload_v3i16_to_v3i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 ; GCN-DAG: ds_write_b32
 ; GCN-DAG: ds_write_b64
@@ -203,6 +241,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_local_sextload_v3i16_to_v3i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 ; GCN-DAG: ds_write_b32
 ; GCN-DAG: ds_write_b64
@@ -221,7 +262,9 @@ entry:
 
 ; FUNC-LABEL: {{^}}local_local_zextload_v4i16_to_v4i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 
 ; EG: LDS_READ_RET
@@ -235,7 +278,9 @@ define amdgpu_kernel void @local_local_zextload_v4i16_to_v4i32(<4 x i32> addrspa
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i16_to_v4i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read_b64
 
 ; EG: LDS_READ_RET
@@ -252,6 +297,9 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i32(<4 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i16_to_v8i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 
 ; EG: LDS_READ_RET
@@ -266,6 +314,9 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i16_to_v8i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 
 ; EG: LDS_READ_RET
@@ -288,6 +339,9 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i16_to_v16i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
 
@@ -312,6 +366,9 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(<16 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i16_to_v16i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
@@ -348,6 +405,9 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(<16 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i16_to_v32i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
@@ -377,6 +437,9 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(<32 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i16_to_v32i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
@@ -414,6 +477,9 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(<32 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v64i16_to_v64i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:14 offset1:15
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3
@@ -479,6 +545,8 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(<64 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v64i16_to_v64i32:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -520,6 +588,9 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(<64 x i32> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i16_to_i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; GCN-DAG: ds_read_u16 v[[LO:[0-9]+]],
 ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
 
@@ -530,7 +601,6 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(<64 x i32> addrspace(
 ; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP
 ; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y
 ; EG-DAG: LDS_WRITE
-; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]]
 define amdgpu_kernel void @local_zextload_i16_to_i64(i64 addrspace(3)* %out, i16 addrspace(3)* %in) #0 {
   %a = load i16, i16 addrspace(3)* %in
   %ext = zext i16 %a to i64
@@ -539,13 +609,16 @@ define amdgpu_kernel void @local_zextload_i16_to_i64(i64 addrspace(3)* %out, i16
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i16_to_i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 ; FIXME: Need to optimize this sequence to avoid an extra shift.
 ;  t25: i32,ch = load<LD2[%in(addrspace=3)], anyext from i16> t12, t10, undef:i32
 ;          t28: i64 = any_extend t25
 ;        t30: i64 = sign_extend_inreg t28, ValueType:ch:i16
 ; SI: ds_read_i16 v[[LO:[0-9]+]],
-; VI: ds_read_u16 v[[ULO:[0-9]+]]
-; VI: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16
+; GFX89: ds_read_u16 v[[ULO:[0-9]+]]
+; GFX89: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16
 ; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 
 ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]]
@@ -566,13 +639,15 @@ define amdgpu_kernel void @local_sextload_i16_to_i64(i64 addrspace(3)* %out, i16
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i16_to_v1i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
 ; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]]
 ; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP
 ; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y
 ; EG-DAG: LDS_WRITE
-; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]]
 define amdgpu_kernel void @local_zextload_v1i16_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i16> addrspace(3)* %in) #0 {
   %load = load <1 x i16>, <1 x i16> addrspace(3)* %in
   %ext = zext <1 x i16> %load to <1 x i64>
@@ -581,6 +656,9 @@ define amdgpu_kernel void @local_zextload_v1i16_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i16_to_v1i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z
 ; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]]
@@ -598,6 +676,9 @@ define amdgpu_kernel void @local_sextload_v1i16_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i16_to_v2i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_zextload_v2i16_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) #0 {
@@ -608,6 +689,9 @@ define amdgpu_kernel void @local_zextload_v2i16_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i16_to_v2i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG-DAG: BFE_INT
@@ -620,6 +704,9 @@ define amdgpu_kernel void @local_sextload_v2i16_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i16_to_v4i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -631,6 +718,9 @@ define amdgpu_kernel void @local_zextload_v4i16_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i16_to_v4i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -646,6 +736,9 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i16_to_v8i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -659,6 +752,9 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i16_to_v8i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -680,6 +776,9 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i16_to_v16i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -697,6 +796,9 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(<16 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i16_to_v16i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -730,6 +832,9 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(<16 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i16_to_v32i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -755,6 +860,9 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(<32 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i16_to_v32i64:
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
+
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
diff --git a/test/CodeGen/AMDGPU/load-local-i32.ll b/test/CodeGen/AMDGPU/load-local-i32.ll
index 86055413d2cf6..c736586fa2172 100644
--- a/test/CodeGen/AMDGPU/load-local-i32.ll
+++ b/test/CodeGen/AMDGPU/load-local-i32.ll
@@ -1,11 +1,12 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
-
 ; FUNC-LABEL: {{^}}local_load_i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0, -1
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -17,6 +18,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2i32:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 ; GCN: ds_read_b64
 define amdgpu_kernel void @local_load_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> addrspace(3)* %in) #0 {
 entry:
@@ -26,6 +30,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i32:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read_b64
 ; GCN-DAG: ds_read_b32
 define amdgpu_kernel void @local_load_v3i32(<3 x i32> addrspace(3)* %out, <3 x i32> addrspace(3)* %in) #0 {
@@ -36,6 +43,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i32:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 
 define amdgpu_kernel void @local_load_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 {
@@ -46,6 +56,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i32:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}}
 define amdgpu_kernel void @local_load_v8i32(<8 x i32> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 {
@@ -56,6 +69,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i32:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5{{$}}
 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}}
@@ -72,6 +88,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i32_to_i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_i32_to_i64(i64 addrspace(3)* %out, i32 addrspace(3)* %in) #0 {
   %ld = load i32, i32 addrspace(3)* %in
   %ext = zext i32 %ld to i64
@@ -80,6 +99,9 @@ define amdgpu_kernel void @local_zextload_i32_to_i64(i64 addrspace(3)* %out, i32
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i32_to_i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_i32_to_i64(i64 addrspace(3)* %out, i32 addrspace(3)* %in) #0 {
   %ld = load i32, i32 addrspace(3)* %in
   %ext = sext i32 %ld to i64
@@ -88,6 +110,9 @@ define amdgpu_kernel void @local_sextload_i32_to_i64(i64 addrspace(3)* %out, i32
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i32_to_v1i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v1i32_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i32> addrspace(3)* %in) #0 {
   %ld = load <1 x i32>, <1 x i32> addrspace(3)* %in
   %ext = zext <1 x i32> %ld to <1 x i64>
@@ -96,6 +121,9 @@ define amdgpu_kernel void @local_zextload_v1i32_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i32_to_v1i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v1i32_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i32> addrspace(3)* %in) #0 {
   %ld = load <1 x i32>, <1 x i32> addrspace(3)* %in
   %ext = sext <1 x i32> %ld to <1 x i64>
@@ -104,6 +132,9 @@ define amdgpu_kernel void @local_sextload_v1i32_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i32_to_v2i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v2i32_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i32> addrspace(3)* %in) #0 {
   %ld = load <2 x i32>, <2 x i32> addrspace(3)* %in
   %ext = zext <2 x i32> %ld to <2 x i64>
@@ -112,6 +143,9 @@ define amdgpu_kernel void @local_zextload_v2i32_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i32_to_v2i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v2i32_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i32> addrspace(3)* %in) #0 {
   %ld = load <2 x i32>, <2 x i32> addrspace(3)* %in
   %ext = sext <2 x i32> %ld to <2 x i64>
@@ -120,6 +154,9 @@ define amdgpu_kernel void @local_sextload_v2i32_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i32_to_v4i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v4i32_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 {
   %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in
   %ext = zext <4 x i32> %ld to <4 x i64>
@@ -128,6 +165,9 @@ define amdgpu_kernel void @local_zextload_v4i32_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i32_to_v4i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v4i32_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 {
   %ld = load <4 x i32>, <4 x i32> addrspace(3)* %in
   %ext = sext <4 x i32> %ld to <4 x i64>
@@ -136,6 +176,9 @@ define amdgpu_kernel void @local_sextload_v4i32_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i32_to_v8i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v8i32_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 {
   %ld = load <8 x i32>, <8 x i32> addrspace(3)* %in
   %ext = zext <8 x i32> %ld to <8 x i64>
@@ -144,6 +187,9 @@ define amdgpu_kernel void @local_zextload_v8i32_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i32_to_v8i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v8i32_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 {
   %ld = load <8 x i32>, <8 x i32> addrspace(3)* %in
   %ext = sext <8 x i32> %ld to <8 x i64>
@@ -152,6 +198,9 @@ define amdgpu_kernel void @local_sextload_v8i32_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i32_to_v16i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v16i32_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i32> addrspace(3)* %in) #0 {
   %ld = load <16 x i32>, <16 x i32> addrspace(3)* %in
   %ext = sext <16 x i32> %ld to <16 x i64>
@@ -160,6 +209,9 @@ define amdgpu_kernel void @local_sextload_v16i32_to_v16i64(<16 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i32_to_v16i64
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v16i32_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i32> addrspace(3)* %in) #0 {
   %ld = load <16 x i32>, <16 x i32> addrspace(3)* %in
   %ext = zext <16 x i32> %ld to <16 x i64>
@@ -168,6 +220,9 @@ define amdgpu_kernel void @local_zextload_v16i32_to_v16i64(<16 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i32_to_v32i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_sextload_v32i32_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i32> addrspace(3)* %in) #0 {
   %ld = load <32 x i32>, <32 x i32> addrspace(3)* %in
   %ext = sext <32 x i32> %ld to <32 x i64>
@@ -176,6 +231,9 @@ define amdgpu_kernel void @local_sextload_v32i32_to_v32i64(<32 x i64> addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i32_to_v32i64:
+; SICIVI: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+
 define amdgpu_kernel void @local_zextload_v32i32_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i32> addrspace(3)* %in) #0 {
   %ld = load <32 x i32>, <32 x i32> addrspace(3)* %in
   %ext = zext <32 x i32> %ld to <32 x i64>
diff --git a/test/CodeGen/AMDGPU/load-local-i64.ll b/test/CodeGen/AMDGPU/load-local-i64.ll
index 0c719a9e0bf90..376f6f513c3a2 100644
--- a/test/CodeGen/AMDGPU/load-local-i64.ll
+++ b/test/CodeGen/AMDGPU/load-local-i64.ll
@@ -1,9 +1,13 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}local_load_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_b64 [[VAL:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}{{$}}
 ; GCN: ds_write_b64 v{{[0-9]+}}, [[VAL]]
 
@@ -16,6 +20,9 @@ define amdgpu_kernel void @local_load_i64(i64 addrspace(3)* %out, i64 addrspace(
 }
 
 ; FUNC-LABEL: {{^}}local_load_v2i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 
 ; EG: LDS_READ_RET
@@ -30,6 +37,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: ds_read2_b64
 ; GCN-DAG: ds_read_b64
 
@@ -47,6 +57,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 
@@ -67,6 +80,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
@@ -96,6 +112,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
 ; GCN: ds_read2_b64
diff --git a/test/CodeGen/AMDGPU/load-local-i8.ll b/test/CodeGen/AMDGPU/load-local-i8.ll
index 16eb366a4b156..72f5408675fcc 100644
--- a/test/CodeGen/AMDGPU/load-local-i8.ll
+++ b/test/CodeGen/AMDGPU/load-local-i8.ll
@@ -1,11 +1,13 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 
 ; FUNC-LABEL: {{^}}local_load_i8:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; GCN: ds_read_u8
 
 ; EG: LDS_UBYTE_READ_RET
@@ -18,7 +20,8 @@ entry:
 
 ; FUNC-LABEL: {{^}}local_load_v2i8:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; GCN: ds_read_u16
 
 ; EG: LDS_USHORT_READ_RET
@@ -30,6 +33,7 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v3i8:
+; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; EG: DS_READ_RET
@@ -41,6 +45,7 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v4i8:
+; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -52,6 +57,7 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v8i8:
+; GFX9-NOT: m0
 ; GCN: ds_read_b64
 
 ; EG: LDS_READ_RET
@@ -64,6 +70,7 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_load_v16i8:
+; GFX9-NOT: m0
 ; GCN: ds_read2_b64  v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}}
 ; GCN: ds_write2_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:{{[0-9]+}}], v[{{[0-9]+}}:[[HI]]{{\]}} offset1:1{{$}}
 
@@ -79,8 +86,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i8_to_i32:
+; GFX9-NOT: m0
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_u8
 
 ; EG: LDS_UBYTE_READ_RET
@@ -93,7 +101,8 @@ define amdgpu_kernel void @local_zextload_i8_to_i32(i32 addrspace(3)* %out, i8 a
 
 ; FUNC-LABEL: {{^}}local_sextload_i8_to_i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_i8
 
 ; EG: LDS_UBYTE_READ_RET
@@ -116,6 +125,7 @@ define amdgpu_kernel void @local_zextload_v1i8_to_v1i32(<1 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i32:
+; GFX9-NOT: m0
 
 ; EG: LDS_UBYTE_READ_RET
 ; EG: BFE_INT
@@ -127,6 +137,7 @@ define amdgpu_kernel void @local_sextload_v1i8_to_v1i32(<1 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i32:
+; GFX9-NOT: m0
 ; GCN: ds_read_u16
 
 ; EG: LDS_USHORT_READ_RET
@@ -139,7 +150,8 @@ define amdgpu_kernel void @local_zextload_v2i8_to_v2i32(<2 x i32> addrspace(3)*
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i8_to_v2i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_u16
 ; FIXME: Need to optimize this sequence to avoid extra shift on VI.
 ;         t23: i16 = srl t39, Constant:i32<8>
@@ -164,6 +176,7 @@ define amdgpu_kernel void @local_sextload_v2i8_to_v2i32(<2 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v3i8_to_v3i32:
+; GFX9-NOT: m0
 ; GCN: ds_read_b32
 
 ; SI-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8
@@ -182,7 +195,8 @@ entry:
 
 ; FUNC-LABEL: {{^}}local_sextload_v3i8_to_v3i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_b32
 
 ; GCN-DAG: v_bfe_i32
@@ -207,7 +221,8 @@ entry:
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i8_to_v4i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_b32
 
 ; EG: LDS_READ_RET
@@ -223,7 +238,8 @@ define amdgpu_kernel void @local_zextload_v4i8_to_v4i32(<4 x i32> addrspace(3)*
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i8_to_v4i32:
 ; GCN-NOT: s_wqm_b64
-; GCN: s_mov_b32 m0
+; GFX9-NOT: m0
+; SICIVI: s_mov_b32 m0
 ; GCN: ds_read_b32
 
 ; EG-DAG: LDS_READ_RET
@@ -239,6 +255,8 @@ define amdgpu_kernel void @local_sextload_v4i8_to_v4i32(<4 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -256,6 +274,8 @@ define amdgpu_kernel void @local_zextload_v8i8_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -275,6 +295,8 @@ define amdgpu_kernel void @local_sextload_v8i8_to_v8i32(<8 x i32> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -300,6 +322,8 @@ define amdgpu_kernel void @local_zextload_v16i8_to_v16i32(<16 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -329,6 +353,8 @@ define amdgpu_kernel void @local_sextload_v16i8_to_v16i32(<16 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -346,6 +372,8 @@ define amdgpu_kernel void @local_zextload_v32i8_to_v32i32(<32 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -363,6 +391,8 @@ define amdgpu_kernel void @local_sextload_v32i8_to_v32i32(<32 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v64i8_to_v64i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -388,6 +418,8 @@ define amdgpu_kernel void @local_zextload_v64i8_to_v64i32(<64 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v64i8_to_v64i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG-DAG: LDS_READ_RET
 ; EG-DAG: LDS_READ_RET
@@ -413,6 +445,9 @@ define amdgpu_kernel void @local_sextload_v64i8_to_v64i32(<64 x i32> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_i8_to_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}}
 ; GCN-DAG: ds_read_u8 v[[LO:[0-9]+]],
 ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]]
@@ -428,6 +463,9 @@ define amdgpu_kernel void @local_zextload_i8_to_i64(i64 addrspace(3)* %out, i8 a
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i8_to_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_read_i8 v[[LO:[0-9]+]],
 ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 
@@ -445,6 +483,8 @@ define amdgpu_kernel void @local_sextload_i8_to_i64(i64 addrspace(3)* %out, i8 a
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i8_to_v1i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_UBYTE_READ_RET
 ; EG: MOV {{.*}}, literal
@@ -458,6 +498,8 @@ define amdgpu_kernel void @local_zextload_v1i8_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_UBYTE_READ_RET
 ; EG: ASHR
@@ -471,6 +513,8 @@ define amdgpu_kernel void @local_sextload_v1i8_to_v1i64(<1 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_USHORT_READ_RET
 define amdgpu_kernel void @local_zextload_v2i8_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 {
@@ -481,6 +525,8 @@ define amdgpu_kernel void @local_zextload_v2i8_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i8_to_v2i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_USHORT_READ_RET
 ; EG: BFE_INT
@@ -493,6 +539,8 @@ define amdgpu_kernel void @local_sextload_v2i8_to_v2i64(<2 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i8_to_v4i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_zextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 {
@@ -503,6 +551,8 @@ define amdgpu_kernel void @local_zextload_v4i8_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i8_to_v4i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 define amdgpu_kernel void @local_sextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 {
@@ -513,6 +563,8 @@ define amdgpu_kernel void @local_sextload_v4i8_to_v4i64(<4 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -524,6 +576,8 @@ define amdgpu_kernel void @local_zextload_v8i8_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -544,6 +598,8 @@ define amdgpu_kernel void @local_sextload_v8i8_to_v8i64(<8 x i64> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -557,6 +613,8 @@ define amdgpu_kernel void @local_zextload_v16i8_to_v16i64(<16 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -570,6 +628,8 @@ define amdgpu_kernel void @local_sextload_v16i8_to_v16i64(<16 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -587,6 +647,8 @@ define amdgpu_kernel void @local_zextload_v32i8_to_v32i64(<32 x i64> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -620,6 +682,8 @@ define amdgpu_kernel void @local_sextload_v32i8_to_v32i64(<32 x i64> addrspace(3
 ; }
 
 ; FUNC-LABEL: {{^}}local_zextload_i8_to_i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; GCN: ds_read_u8 v[[VAL:[0-9]+]],
 ; GCN: ds_write_b16 v[[VAL:[0-9]+]]
 
@@ -633,6 +697,8 @@ define amdgpu_kernel void @local_zextload_i8_to_i16(i16 addrspace(3)* %out, i8 a
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_i8_to_i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 ; GCN: ds_read_i8 v[[VAL:[0-9]+]],
 ; GCN: ds_write_b16 v{{[0-9]+}}, v[[VAL]]
 
@@ -647,6 +713,8 @@ define amdgpu_kernel void @local_sextload_i8_to_i16(i16 addrspace(3)* %out, i8 a
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v1i8_to_v1i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_UBYTE_READ_RET
 ; EG: LDS_SHORT_WRITE
@@ -658,6 +726,8 @@ define amdgpu_kernel void @local_zextload_v1i8_to_v1i16(<1 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_UBYTE_READ_RET
 ; EG: BFE_INT
@@ -670,6 +740,8 @@ define amdgpu_kernel void @local_sextload_v1i8_to_v1i16(<1 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_USHORT_READ_RET
 ; EG: LDS_WRITE
@@ -681,6 +753,8 @@ define amdgpu_kernel void @local_zextload_v2i8_to_v2i16(<2 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v2i8_to_v2i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_USHORT_READ_RET
 ; EG: BFE_INT
@@ -694,6 +768,8 @@ define amdgpu_kernel void @local_sextload_v2i8_to_v2i16(<2 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v4i8_to_v4i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_WRITE
@@ -706,6 +782,8 @@ define amdgpu_kernel void @local_zextload_v4i8_to_v4i16(<4 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v4i8_to_v4i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; TODO: these do LSHR + BFE_INT, instead of just BFE_INT/ASHR
@@ -723,6 +801,8 @@ define amdgpu_kernel void @local_sextload_v4i8_to_v4i16(<4 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -738,6 +818,8 @@ define amdgpu_kernel void @local_zextload_v8i8_to_v8i16(<8 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -762,6 +844,8 @@ define amdgpu_kernel void @local_sextload_v8i8_to_v8i16(<8 x i16> addrspace(3)*
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -783,6 +867,8 @@ define amdgpu_kernel void @local_zextload_v16i8_to_v16i16(<16 x i16> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -821,6 +907,8 @@ define amdgpu_kernel void @local_sextload_v16i8_to_v16i16(<16 x i16> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
@@ -854,6 +942,8 @@ define amdgpu_kernel void @local_zextload_v32i8_to_v32i16(<32 x i16> addrspace(3
 }
 
 ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
 
 ; EG: LDS_READ_RET
 ; EG: LDS_READ_RET
diff --git a/test/CodeGen/AMDGPU/load-private-double16-amdgiz.ll b/test/CodeGen/AMDGPU/load-private-double16-amdgiz.ll
new file mode 100644
index 0000000000000..e158677e0982b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/load-private-double16-amdgiz.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5"
+
+; GCN-LABEL: @test_unaligned_load
+; GCN: buffer_load_dword
+; GCN-NOT: flat_load_dword
+define amdgpu_kernel void @test_unaligned_load(<16 x double> addrspace(1)* %results, i32 %i) {
+entry:
+  %a = inttoptr i32 %i to <16 x double> addrspace(5)*
+  %v = load <16 x double>, <16 x double> addrspace(5)* %a, align 8 
+  store <16 x double> %v, <16 x double> addrspace(1)* %results, align 128
+  ret void
+}
+
+; GCN-LABEL: @test_unaligned_store
+; GCN: buffer_store_dword
+; GCN-NOT: flat_store_dword
+define amdgpu_kernel void @test_unaligned_store(<16 x double> %v, i32 %i) {
+entry:
+  %a = inttoptr i32 %i to <16 x double> addrspace(5)*
+  store <16 x double> %v, <16 x double> addrspace(5)* %a, align 8
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/load-weird-sizes.ll b/test/CodeGen/AMDGPU/load-weird-sizes.ll
index f9ba6241fe067..fff0d3520d742 100644
--- a/test/CodeGen/AMDGPU/load-weird-sizes.ll
+++ b/test/CodeGen/AMDGPU/load-weird-sizes.ll
@@ -5,8 +5,8 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=R600 -check-prefix=CM -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}load_i24:
-; SI: {{flat|buffer}}_load_ubyte
-; SI: {{flat|buffer}}_load_ushort
+; SI-DAG: {{flat|buffer}}_load_ubyte
+; SI-DAG: {{flat|buffer}}_load_ushort
 ; SI: {{flat|buffer}}_store_dword
 define amdgpu_kernel void @load_i24(i32 addrspace(1)* %out, i24 addrspace(1)* %in) #0 {
   %1 = load i24, i24 addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/local-64.ll b/test/CodeGen/AMDGPU/local-64.ll
index bf4a93237bd4c..87c18a7fc449f 100644
--- a/test/CodeGen/AMDGPU/local-64.ll
+++ b/test/CodeGen/AMDGPU/local-64.ll
@@ -1,10 +1,14 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck --check-prefix=SI --check-prefix=BOTH %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs< %s | FileCheck --check-prefix=CI --check-prefix=BOTH %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck --check-prefix=CI --check-prefix=BOTH %s
+; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
-; BOTH-LABEL: {{^}}local_i32_load
-; BOTH: ds_read_b32 [[REG:v[0-9]+]], v{{[0-9]+}} offset:28
-; BOTH: buffer_store_dword [[REG]],
+; GCN-LABEL: {{^}}local_i32_load
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read_b32 [[REG:v[0-9]+]], v{{[0-9]+}} offset:28
+; GCN: buffer_store_dword [[REG]],
 define amdgpu_kernel void @local_i32_load(i32 addrspace(1)* %out, i32 addrspace(3)* %in) nounwind {
   %gep = getelementptr i32, i32 addrspace(3)* %in, i32 7
   %val = load i32, i32 addrspace(3)* %gep, align 4
@@ -12,19 +16,25 @@ define amdgpu_kernel void @local_i32_load(i32 addrspace(1)* %out, i32 addrspace(
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i32_load_0_offset
-; BOTH: ds_read_b32 [[REG:v[0-9]+]], v{{[0-9]+}}
-; BOTH: buffer_store_dword [[REG]],
+; GCN-LABEL: {{^}}local_i32_load_0_offset
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read_b32 [[REG:v[0-9]+]], v{{[0-9]+}}
+; GCN: buffer_store_dword [[REG]],
 define amdgpu_kernel void @local_i32_load_0_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %in) nounwind {
   %val = load i32, i32 addrspace(3)* %in, align 4
   store i32 %val, i32 addrspace(1)* %out, align 4
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i8_load_i16_max_offset:
-; BOTH-NOT: ADD
-; BOTH: ds_read_u8 [[REG:v[0-9]+]], {{v[0-9]+}} offset:65535
-; BOTH: buffer_store_byte [[REG]],
+; GCN-LABEL: {{^}}local_i8_load_i16_max_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_read_u8 [[REG:v[0-9]+]], {{v[0-9]+}} offset:65535
+; GCN: buffer_store_byte [[REG]],
 define amdgpu_kernel void @local_i8_load_i16_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)* %in) nounwind {
   %gep = getelementptr i8, i8 addrspace(3)* %in, i32 65535
   %val = load i8, i8 addrspace(3)* %gep, align 4
@@ -32,14 +42,20 @@ define amdgpu_kernel void @local_i8_load_i16_max_offset(i8 addrspace(1)* %out, i
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i8_load_over_i16_max_offset:
+; GCN-LABEL: {{^}}local_i8_load_over_i16_max_offset:
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; The LDS offset will be 65536 bytes, which is larger than the size of LDS on
 ; SI, which is why it is being OR'd with the base pointer.
-; SI: s_or_b32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
-; CI: s_add_i32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
-; BOTH: v_mov_b32_e32 [[VREGADDR:v[0-9]+]], [[ADDR]]
-; BOTH: ds_read_u8 [[REG:v[0-9]+]], [[VREGADDR]]
-; BOTH: buffer_store_byte [[REG]],
+; SI-DAG: s_or_b32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
+; CI-DAG: s_add_i32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
+; VI-DAG: s_add_i32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
+; GFX9-DAG: s_add_i32 [[ADDR:s[0-9]+]], s{{[0-9]+}}, 0x10000
+
+; GCN-DAG: v_mov_b32_e32 [[VREGADDR:v[0-9]+]], [[ADDR]]
+; GCN: ds_read_u8 [[REG:v[0-9]+]], [[VREGADDR]]
+; GCN: buffer_store_byte [[REG]],
 define amdgpu_kernel void @local_i8_load_over_i16_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)* %in) nounwind {
   %gep = getelementptr i8, i8 addrspace(3)* %in, i32 65536
   %val = load i8, i8 addrspace(3)* %gep, align 4
@@ -47,10 +63,13 @@ define amdgpu_kernel void @local_i8_load_over_i16_max_offset(i8 addrspace(1)* %o
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i64_load:
-; BOTH-NOT: ADD
-; BOTH: ds_read_b64 [[REG:v[[0-9]+:[0-9]+]]], v{{[0-9]+}} offset:56
-; BOTH: buffer_store_dwordx2 [[REG]],
+; GCN-LABEL: {{^}}local_i64_load:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_read_b64 [[REG:v[[0-9]+:[0-9]+]]], v{{[0-9]+}} offset:56
+; GCN: buffer_store_dwordx2 [[REG]],
 define amdgpu_kernel void @local_i64_load(i64 addrspace(1)* %out, i64 addrspace(3)* %in) nounwind {
   %gep = getelementptr i64, i64 addrspace(3)* %in, i32 7
   %val = load i64, i64 addrspace(3)* %gep, align 8
@@ -58,19 +77,25 @@ define amdgpu_kernel void @local_i64_load(i64 addrspace(1)* %out, i64 addrspace(
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i64_load_0_offset
-; BOTH: ds_read_b64 [[REG:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
-; BOTH: buffer_store_dwordx2 [[REG]],
+; GCN-LABEL: {{^}}local_i64_load_0_offset
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read_b64 [[REG:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
+; GCN: buffer_store_dwordx2 [[REG]],
 define amdgpu_kernel void @local_i64_load_0_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %in) nounwind {
   %val = load i64, i64 addrspace(3)* %in, align 8
   store i64 %val, i64 addrspace(1)* %out, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_f64_load:
-; BOTH-NOT: ADD
-; BOTH: ds_read_b64 [[REG:v[[0-9]+:[0-9]+]]], v{{[0-9]+}} offset:56
-; BOTH: buffer_store_dwordx2 [[REG]],
+; GCN-LABEL: {{^}}local_f64_load:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_read_b64 [[REG:v[[0-9]+:[0-9]+]]], v{{[0-9]+}} offset:56
+; GCN: buffer_store_dwordx2 [[REG]],
 define amdgpu_kernel void @local_f64_load(double addrspace(1)* %out, double addrspace(3)* %in) nounwind {
   %gep = getelementptr double, double addrspace(3)* %in, i32 7
   %val = load double, double addrspace(3)* %gep, align 8
@@ -78,83 +103,110 @@ define amdgpu_kernel void @local_f64_load(double addrspace(1)* %out, double addr
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_f64_load_0_offset
-; BOTH: ds_read_b64 [[REG:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
-; BOTH: buffer_store_dwordx2 [[REG]],
+; GCN-LABEL: {{^}}local_f64_load_0_offset
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_read_b64 [[REG:v\[[0-9]+:[0-9]+\]]], v{{[0-9]+}}
+; GCN: buffer_store_dwordx2 [[REG]],
 define amdgpu_kernel void @local_f64_load_0_offset(double addrspace(1)* %out, double addrspace(3)* %in) nounwind {
   %val = load double, double addrspace(3)* %in, align 8
   store double %val, double addrspace(1)* %out, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i64_store:
-; BOTH-NOT: ADD
-; BOTH: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:56
+; GCN-LABEL: {{^}}local_i64_store:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:56
 define amdgpu_kernel void @local_i64_store(i64 addrspace(3)* %out) nounwind {
   %gep = getelementptr i64, i64 addrspace(3)* %out, i32 7
   store i64 5678, i64 addrspace(3)* %gep, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_i64_store_0_offset:
-; BOTH-NOT: ADD
-; BOTH: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
+; GCN-LABEL: {{^}}local_i64_store_0_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @local_i64_store_0_offset(i64 addrspace(3)* %out) nounwind {
   store i64 1234, i64 addrspace(3)* %out, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_f64_store:
-; BOTH-NOT: ADD
-; BOTH: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:56
+; GCN-LABEL: {{^}}local_f64_store:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:56
 define amdgpu_kernel void @local_f64_store(double addrspace(3)* %out) nounwind {
   %gep = getelementptr double, double addrspace(3)* %out, i32 7
   store double 16.0, double addrspace(3)* %gep, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_f64_store_0_offset
-; BOTH: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
+; GCN-LABEL: {{^}}local_f64_store_0_offset
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @local_f64_store_0_offset(double addrspace(3)* %out) nounwind {
   store double 20.0, double addrspace(3)* %out, align 8
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_v2i64_store:
-; BOTH-NOT: ADD
-; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15
-; BOTH: s_endpgm
+; GCN-LABEL: {{^}}local_v2i64_store:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15
+; GCN: s_endpgm
 define amdgpu_kernel void @local_v2i64_store(<2 x i64> addrspace(3)* %out) nounwind {
   %gep = getelementptr <2 x i64>, <2 x i64> addrspace(3)* %out, i32 7
   store <2 x i64> <i64 5678, i64 5678>, <2 x i64> addrspace(3)* %gep, align 16
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_v2i64_store_0_offset:
-; BOTH-NOT: ADD
-; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
-; BOTH: s_endpgm
+; GCN-LABEL: {{^}}local_v2i64_store_0_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
+; GCN: s_endpgm
 define amdgpu_kernel void @local_v2i64_store_0_offset(<2 x i64> addrspace(3)* %out) nounwind {
   store <2 x i64> <i64 1234, i64 1234>, <2 x i64> addrspace(3)* %out, align 16
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_v4i64_store:
-; BOTH-NOT: ADD
-; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31
-; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29
-; BOTH: s_endpgm
+; GCN-LABEL: {{^}}local_v4i64_store:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31
+; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29
+; GCN: s_endpgm
 define amdgpu_kernel void @local_v4i64_store(<4 x i64> addrspace(3)* %out) nounwind {
   %gep = getelementptr <4 x i64>, <4 x i64> addrspace(3)* %out, i32 7
   store <4 x i64> <i64 5678, i64 5678, i64 5678, i64 5678>, <4 x i64> addrspace(3)* %gep, align 16
   ret void
 }
 
-; BOTH-LABEL: {{^}}local_v4i64_store_0_offset:
-; BOTH-NOT: ADD
-; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
-; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
-; BOTH: s_endpgm
+; GCN-LABEL: {{^}}local_v4i64_store_0_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-NOT: add
+; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3
+; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1
+; GCN: s_endpgm
 define amdgpu_kernel void @local_v4i64_store_0_offset(<4 x i64> addrspace(3)* %out) nounwind {
   store <4 x i64> <i64 1234, i64 1234, i64 1234, i64 1234>, <4 x i64> addrspace(3)* %out, align 16
   ret void
diff --git a/test/CodeGen/AMDGPU/local-atomics.ll b/test/CodeGen/AMDGPU/local-atomics.ll
index de029d964b0d5..d2167f5a730a2 100644
--- a/test/CodeGen/AMDGPU/local-atomics.ll
+++ b/test/CodeGen/AMDGPU/local-atomics.ll
@@ -1,13 +1,18 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CIVI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=CIVI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_i32:
 ; EG: LDS_WRXCHG_RET *
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
@@ -18,6 +23,9 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i32(i32 addrspace(1)* %out, i32 a
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRXCHG_RET *
 ; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -31,9 +39,13 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i32_offset(i32 addrspace(1)* %out
 ; XXX - Is it really necessary to load 4 into VGPR?
 ; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32:
 ; EG: LDS_ADD_RET *
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_add_rtn_u32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
 ; GCN: buffer_store_dword [[RESULT]],
 ; GCN: s_endpgm
@@ -44,6 +56,9 @@ define amdgpu_kernel void @lds_atomic_add_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_ADD_RET *
 ; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -55,6 +70,9 @@ define amdgpu_kernel void @lds_atomic_add_ret_i32_offset(i32 addrspace(1)* %out,
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32_bad_si_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_ADD_RET *
 ; SI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CIVI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
@@ -70,7 +88,11 @@ define amdgpu_kernel void @lds_atomic_add_ret_i32_bad_si_offset(i32 addrspace(1)
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32:
 ; EG: LDS_ADD_RET *
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -81,7 +103,11 @@ define amdgpu_kernel void @lds_atomic_add1_ret_i32(i32 addrspace(1)* %out, i32 a
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32_offset:
 ; EG: LDS_ADD_RET *
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]] offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -92,6 +118,9 @@ define amdgpu_kernel void @lds_atomic_add1_ret_i32_offset(i32 addrspace(1)* %out
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_ret_i32_bad_si_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_ADD_RET *
 ; SI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CIVI: ds_add_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
@@ -107,6 +136,10 @@ define amdgpu_kernel void @lds_atomic_add1_ret_i32_bad_si_offset(i32 addrspace(1
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub_ret_i32:
 ; EG: LDS_SUB_RET *
+
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_rtn_u32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -117,6 +150,10 @@ define amdgpu_kernel void @lds_atomic_sub_ret_i32(i32 addrspace(1)* %out, i32 ad
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub_ret_i32_offset:
 ; EG: LDS_SUB_RET *
+
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -128,7 +165,11 @@ define amdgpu_kernel void @lds_atomic_sub_ret_i32_offset(i32 addrspace(1)* %out,
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub1_ret_i32:
 ; EG: LDS_SUB_RET *
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_sub_rtn_u32  v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -139,7 +180,11 @@ define amdgpu_kernel void @lds_atomic_sub1_ret_i32(i32 addrspace(1)* %out, i32 a
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub1_ret_i32_offset:
 ; EG: LDS_SUB_RET *
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_sub_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, [[ONE]] offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_ret_i32_offset(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -151,6 +196,10 @@ define amdgpu_kernel void @lds_atomic_sub1_ret_i32_offset(i32 addrspace(1)* %out
 
 ; FUNC-LABEL: {{^}}lds_atomic_and_ret_i32:
 ; EG: LDS_AND_RET *
+
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_rtn_b32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_ret_i32(i32 addrspace(1)* %out, i32 addrspace(3)* %ptr) nounwind {
@@ -160,6 +209,9 @@ define amdgpu_kernel void @lds_atomic_and_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_and_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_AND_RET *
 ; GCN: ds_and_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -171,6 +223,9 @@ define amdgpu_kernel void @lds_atomic_and_ret_i32_offset(i32 addrspace(1)* %out,
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_or_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_OR_RET *
 ; GCN: ds_or_rtn_b32
 ; GCN: s_endpgm
@@ -181,6 +236,9 @@ define amdgpu_kernel void @lds_atomic_or_ret_i32(i32 addrspace(1)* %out, i32 add
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_or_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_OR_RET *
 ; GCN: ds_or_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -192,6 +250,9 @@ define amdgpu_kernel void @lds_atomic_or_ret_i32_offset(i32 addrspace(1)* %out,
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xor_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_XOR_RET *
 ; GCN: ds_xor_rtn_b32
 ; GCN: s_endpgm
@@ -202,6 +263,9 @@ define amdgpu_kernel void @lds_atomic_xor_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xor_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_XOR_RET *
 ; GCN: ds_xor_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -221,6 +285,9 @@ define amdgpu_kernel void @lds_atomic_xor_ret_i32_offset(i32 addrspace(1)* %out,
 ; }
 
 ; FUNC-LABEL: {{^}}lds_atomic_min_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MIN_INT_RET *
 ; GCN: ds_min_rtn_i32
 ; GCN: s_endpgm
@@ -231,6 +298,9 @@ define amdgpu_kernel void @lds_atomic_min_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_min_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MIN_INT_RET *
 ; GCN: ds_min_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -242,6 +312,9 @@ define amdgpu_kernel void @lds_atomic_min_ret_i32_offset(i32 addrspace(1)* %out,
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_max_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MAX_INT_RET *
 ; GCN: ds_max_rtn_i32
 ; GCN: s_endpgm
@@ -252,6 +325,9 @@ define amdgpu_kernel void @lds_atomic_max_ret_i32(i32 addrspace(1)* %out, i32 ad
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_max_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MAX_INT_RET *
 ; GCN: ds_max_rtn_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -263,6 +339,9 @@ define amdgpu_kernel void @lds_atomic_max_ret_i32_offset(i32 addrspace(1)* %out,
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umin_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MIN_UINT_RET *
 ; GCN: ds_min_rtn_u32
 ; GCN: s_endpgm
@@ -273,6 +352,9 @@ define amdgpu_kernel void @lds_atomic_umin_ret_i32(i32 addrspace(1)* %out, i32 a
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umin_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MIN_UINT_RET *
 ; GCN: ds_min_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -284,6 +366,9 @@ define amdgpu_kernel void @lds_atomic_umin_ret_i32_offset(i32 addrspace(1)* %out
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umax_ret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MAX_UINT_RET *
 ; GCN: ds_max_rtn_u32
 ; GCN: s_endpgm
@@ -294,6 +379,9 @@ define amdgpu_kernel void @lds_atomic_umax_ret_i32(i32 addrspace(1)* %out, i32 a
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umax_ret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_MAX_UINT_RET *
 ; GCN: ds_max_rtn_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -305,9 +393,12 @@ define amdgpu_kernel void @lds_atomic_umax_ret_i32_offset(i32 addrspace(1)* %out
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xchg_noret_i32:
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -316,6 +407,9 @@ define amdgpu_kernel void @lds_atomic_xchg_noret_i32(i32 addrspace(3)* %ptr) nou
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xchg_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -325,9 +419,12 @@ define amdgpu_kernel void @lds_atomic_xchg_noret_i32_offset(i32 addrspace(3)* %p
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32:
-; GCN: s_load_dword [[SPTR:s[0-9]+]],
-; GCN: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
 ; GCN: ds_add_u32 [[VPTR]], [[DATA]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -336,6 +433,9 @@ define amdgpu_kernel void @lds_atomic_add_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -345,6 +445,9 @@ define amdgpu_kernel void @lds_atomic_add_noret_i32_offset(i32 addrspace(3)* %pt
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add_noret_i32_bad_si_offset
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; SI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}}
 ; CIVI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -357,7 +460,10 @@ define amdgpu_kernel void @lds_atomic_add_noret_i32_bad_si_offset(i32 addrspace(
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32:
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_add_u32 v{{[0-9]+}}, [[ONE]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -366,7 +472,10 @@ define amdgpu_kernel void @lds_atomic_add1_noret_i32(i32 addrspace(3)* %ptr) nou
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32_offset:
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_add_u32 v{{[0-9]+}}, [[ONE]] offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -376,6 +485,9 @@ define amdgpu_kernel void @lds_atomic_add1_noret_i32_offset(i32 addrspace(3)* %p
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_add1_noret_i32_bad_si_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; SI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}}
 ; CIVI: ds_add_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
@@ -388,6 +500,9 @@ define amdgpu_kernel void @lds_atomic_add1_noret_i32_bad_si_offset(i32 addrspace
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_u32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -396,6 +511,9 @@ define amdgpu_kernel void @lds_atomic_sub_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -405,7 +523,10 @@ define amdgpu_kernel void @lds_atomic_sub_noret_i32_offset(i32 addrspace(3)* %pt
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub1_noret_i32:
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_sub_u32 v{{[0-9]+}}, [[ONE]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -414,7 +535,10 @@ define amdgpu_kernel void @lds_atomic_sub1_noret_i32(i32 addrspace(3)* %ptr) nou
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_sub1_noret_i32_offset:
-; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 [[ONE:v[0-9]+]], 1{{$}}
 ; GCN: ds_sub_u32 v{{[0-9]+}}, [[ONE]] offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -424,6 +548,9 @@ define amdgpu_kernel void @lds_atomic_sub1_noret_i32_offset(i32 addrspace(3)* %p
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_and_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_b32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -432,6 +559,9 @@ define amdgpu_kernel void @lds_atomic_and_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_and_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -441,6 +571,9 @@ define amdgpu_kernel void @lds_atomic_and_noret_i32_offset(i32 addrspace(3)* %pt
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_or_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_b32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -449,6 +582,9 @@ define amdgpu_kernel void @lds_atomic_or_noret_i32(i32 addrspace(3)* %ptr) nounw
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_or_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -458,6 +594,9 @@ define amdgpu_kernel void @lds_atomic_or_noret_i32_offset(i32 addrspace(3)* %ptr
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xor_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_b32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -466,6 +605,9 @@ define amdgpu_kernel void @lds_atomic_xor_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_xor_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -482,6 +624,9 @@ define amdgpu_kernel void @lds_atomic_xor_noret_i32_offset(i32 addrspace(3)* %pt
 ; }
 
 ; FUNC-LABEL: {{^}}lds_atomic_min_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_i32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -490,6 +635,9 @@ define amdgpu_kernel void @lds_atomic_min_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_min_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_i32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -499,6 +647,9 @@ define amdgpu_kernel void @lds_atomic_min_noret_i32_offset(i32 addrspace(3)* %pt
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_max_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_i32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -507,6 +658,9 @@ define amdgpu_kernel void @lds_atomic_max_noret_i32(i32 addrspace(3)* %ptr) noun
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_max_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_i32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -516,6 +670,9 @@ define amdgpu_kernel void @lds_atomic_max_noret_i32_offset(i32 addrspace(3)* %pt
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umin_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_u32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -524,6 +681,9 @@ define amdgpu_kernel void @lds_atomic_umin_noret_i32(i32 addrspace(3)* %ptr) nou
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umin_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
@@ -533,6 +693,9 @@ define amdgpu_kernel void @lds_atomic_umin_noret_i32_offset(i32 addrspace(3)* %p
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umax_noret_i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_u32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_noret_i32(i32 addrspace(3)* %ptr) nounwind {
@@ -541,6 +704,9 @@ define amdgpu_kernel void @lds_atomic_umax_noret_i32(i32 addrspace(3)* %ptr) nou
 }
 
 ; FUNC-LABEL: {{^}}lds_atomic_umax_noret_i32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_u32 v{{[0-9]+}}, v{{[0-9]+}} offset:16
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_noret_i32_offset(i32 addrspace(3)* %ptr) nounwind {
diff --git a/test/CodeGen/AMDGPU/local-atomics64.ll b/test/CodeGen/AMDGPU/local-atomics64.ll
index 6572a7bcd4fe2..6155bfcf1aef6 100644
--- a/test/CodeGen/AMDGPU/local-atomics64.ll
+++ b/test/CodeGen/AMDGPU/local-atomics64.ll
@@ -1,7 +1,11 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=SI -check-prefix=GCN %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=VI -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,SI,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,VI,SICIVI,GFX89 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9,GFX89 %s
 
 ; GCN-LABEL: {{^}}lds_atomic_xchg_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_wrxchg_rtn_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -11,6 +15,9 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i64(i64 addrspace(1)* %out, i64 a
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xchg_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -21,6 +28,9 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i64_offset(i64 addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_add_rtn_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -30,10 +40,13 @@ define amdgpu_kernel void @lds_atomic_add_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add_ret_i64_offset:
-; SI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; VI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GCN: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
-; GCN: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
+; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
 ; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
 ; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}} offset:32
 ; GCN: buffer_store_dwordx2 [[RESULT]],
@@ -46,9 +59,12 @@ define amdgpu_kernel void @lds_atomic_add_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add1_ret_i64:
-; GCN: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
-; GCN: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
-; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
+; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
 ; GCN: buffer_store_dwordx2 [[RESULT]],
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -58,6 +74,9 @@ define amdgpu_kernel void @lds_atomic_add1_ret_i64(i64 addrspace(1)* %out, i64 a
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add1_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_add_rtn_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -68,6 +87,9 @@ define amdgpu_kernel void @lds_atomic_add1_ret_i64_offset(i64 addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_rtn_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -77,6 +99,9 @@ define amdgpu_kernel void @lds_atomic_sub_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_rtn_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -87,9 +112,12 @@ define amdgpu_kernel void @lds_atomic_sub_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub1_ret_i64:
-; GCN: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
-; GCN: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
-; GCN: ds_sub_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
+; GCN: ds_sub_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
 ; GCN: buffer_store_dwordx2 [[RESULT]],
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -99,6 +127,9 @@ define amdgpu_kernel void @lds_atomic_sub1_ret_i64(i64 addrspace(1)* %out, i64 a
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub1_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_rtn_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -109,6 +140,9 @@ define amdgpu_kernel void @lds_atomic_sub1_ret_i64_offset(i64 addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_and_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_rtn_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -118,6 +152,9 @@ define amdgpu_kernel void @lds_atomic_and_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_and_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_rtn_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -128,6 +165,9 @@ define amdgpu_kernel void @lds_atomic_and_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_or_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_rtn_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -137,6 +177,9 @@ define amdgpu_kernel void @lds_atomic_or_ret_i64(i64 addrspace(1)* %out, i64 add
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_or_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_rtn_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -147,6 +190,9 @@ define amdgpu_kernel void @lds_atomic_or_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xor_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_rtn_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -156,6 +202,9 @@ define amdgpu_kernel void @lds_atomic_xor_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xor_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_rtn_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -174,6 +223,9 @@ define amdgpu_kernel void @lds_atomic_xor_ret_i64_offset(i64 addrspace(1)* %out,
 ; }
 
 ; GCN-LABEL: {{^}}lds_atomic_min_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_rtn_i64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -183,6 +235,9 @@ define amdgpu_kernel void @lds_atomic_min_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_min_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_rtn_i64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -193,6 +248,9 @@ define amdgpu_kernel void @lds_atomic_min_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_max_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_rtn_i64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -202,6 +260,9 @@ define amdgpu_kernel void @lds_atomic_max_ret_i64(i64 addrspace(1)* %out, i64 ad
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_max_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_rtn_i64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -212,6 +273,9 @@ define amdgpu_kernel void @lds_atomic_max_ret_i64_offset(i64 addrspace(1)* %out,
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umin_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_rtn_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -221,6 +285,9 @@ define amdgpu_kernel void @lds_atomic_umin_ret_i64(i64 addrspace(1)* %out, i64 a
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umin_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_rtn_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -231,6 +298,9 @@ define amdgpu_kernel void @lds_atomic_umin_ret_i64_offset(i64 addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umax_ret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_rtn_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_ret_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -240,6 +310,9 @@ define amdgpu_kernel void @lds_atomic_umax_ret_i64(i64 addrspace(1)* %out, i64 a
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umax_ret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_rtn_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_ret_i64_offset(i64 addrspace(1)* %out, i64 addrspace(3)* %ptr) nounwind {
@@ -250,6 +323,9 @@ define amdgpu_kernel void @lds_atomic_umax_ret_i64_offset(i64 addrspace(1)* %out
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xchg_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_wrxchg_rtn_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -258,6 +334,9 @@ define amdgpu_kernel void @lds_atomic_xchg_noret_i64(i64 addrspace(3)* %ptr) nou
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xchg_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xchg_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -267,6 +346,9 @@ define amdgpu_kernel void @lds_atomic_xchg_noret_i64_offset(i64 addrspace(3)* %p
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_add_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -275,12 +357,15 @@ define amdgpu_kernel void @lds_atomic_add_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add_noret_i64_offset:
-; SI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
-; VI: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; GCN: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
-; GCN: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
+; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
+; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
 ; GCN: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
-; GCN: ds_add_u64 [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}} offset:32
+; GCN: ds_add_u64 {{v[0-9]+}}, v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
   %gep = getelementptr i64, i64 addrspace(3)* %ptr, i64 4
@@ -289,9 +374,12 @@ define amdgpu_kernel void @lds_atomic_add_noret_i64_offset(i64 addrspace(3)* %pt
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add1_noret_i64:
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
 ; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
-; GCN: ds_add_u64 [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
+; GCN: ds_add_u64 {{v[0-9]+}}, v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_noret_i64(i64 addrspace(3)* %ptr) nounwind {
   %result = atomicrmw add i64 addrspace(3)* %ptr, i64 1 seq_cst
@@ -299,6 +387,9 @@ define amdgpu_kernel void @lds_atomic_add1_noret_i64(i64 addrspace(3)* %ptr) nou
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_add1_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_add_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_add1_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -308,6 +399,9 @@ define amdgpu_kernel void @lds_atomic_add1_noret_i64_offset(i64 addrspace(3)* %p
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -316,6 +410,9 @@ define amdgpu_kernel void @lds_atomic_sub_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -325,9 +422,12 @@ define amdgpu_kernel void @lds_atomic_sub_noret_i64_offset(i64 addrspace(3)* %pt
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub1_noret_i64:
-; GCN: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
-; GCN: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
-; GCN: ds_sub_u64 [[VPTR]], v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
+; SICIVI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 1{{$}}
+; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0{{$}}
+; GCN: ds_sub_u64 {{v[0-9]+}}, v{{\[}}[[LOVDATA]]:[[HIVDATA]]{{\]}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_noret_i64(i64 addrspace(3)* %ptr) nounwind {
   %result = atomicrmw sub i64 addrspace(3)* %ptr, i64 1 seq_cst
@@ -335,6 +435,9 @@ define amdgpu_kernel void @lds_atomic_sub1_noret_i64(i64 addrspace(3)* %ptr) nou
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_sub1_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_sub_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_sub1_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -344,6 +447,9 @@ define amdgpu_kernel void @lds_atomic_sub1_noret_i64_offset(i64 addrspace(3)* %p
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_and_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -352,6 +458,9 @@ define amdgpu_kernel void @lds_atomic_and_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_and_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_and_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_and_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -361,6 +470,9 @@ define amdgpu_kernel void @lds_atomic_and_noret_i64_offset(i64 addrspace(3)* %pt
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_or_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -369,6 +481,9 @@ define amdgpu_kernel void @lds_atomic_or_noret_i64(i64 addrspace(3)* %ptr) nounw
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_or_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_or_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_or_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -378,6 +493,9 @@ define amdgpu_kernel void @lds_atomic_or_noret_i64_offset(i64 addrspace(3)* %ptr
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xor_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_b64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -386,6 +504,9 @@ define amdgpu_kernel void @lds_atomic_xor_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_xor_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_xor_b64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_xor_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -402,6 +523,9 @@ define amdgpu_kernel void @lds_atomic_xor_noret_i64_offset(i64 addrspace(3)* %pt
 ; }
 
 ; GCN-LABEL: {{^}}lds_atomic_min_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_i64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -410,6 +534,9 @@ define amdgpu_kernel void @lds_atomic_min_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_min_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_i64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_min_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -419,6 +546,9 @@ define amdgpu_kernel void @lds_atomic_min_noret_i64_offset(i64 addrspace(3)* %pt
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_max_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_i64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -427,6 +557,9 @@ define amdgpu_kernel void @lds_atomic_max_noret_i64(i64 addrspace(3)* %ptr) noun
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_max_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_i64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_max_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -436,6 +569,9 @@ define amdgpu_kernel void @lds_atomic_max_noret_i64_offset(i64 addrspace(3)* %pt
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umin_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -444,6 +580,9 @@ define amdgpu_kernel void @lds_atomic_umin_noret_i64(i64 addrspace(3)* %ptr) nou
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umin_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_min_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umin_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
@@ -453,6 +592,9 @@ define amdgpu_kernel void @lds_atomic_umin_noret_i64_offset(i64 addrspace(3)* %p
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umax_noret_i64:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_u64
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_noret_i64(i64 addrspace(3)* %ptr) nounwind {
@@ -461,6 +603,9 @@ define amdgpu_kernel void @lds_atomic_umax_noret_i64(i64 addrspace(3)* %ptr) nou
 }
 
 ; GCN-LABEL: {{^}}lds_atomic_umax_noret_i64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; GCN: ds_max_u64 {{.*}} offset:32
 ; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_umax_noret_i64_offset(i64 addrspace(3)* %ptr) nounwind {
diff --git a/test/CodeGen/AMDGPU/loop-idiom.ll b/test/CodeGen/AMDGPU/loop-idiom.ll
index 23ddd6488af99..d783eaa0e4a1e 100644
--- a/test/CodeGen/AMDGPU/loop-idiom.ll
+++ b/test/CodeGen/AMDGPU/loop-idiom.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -basicaa -loop-idiom -S < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s
-; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=SI -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ; RUN: opt -basicaa -loop-idiom -S < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s
 
 
diff --git a/test/CodeGen/AMDGPU/loop_break.ll b/test/CodeGen/AMDGPU/loop_break.ll
index b9df2cb779ad0..b2641cd4d2e41 100644
--- a/test/CodeGen/AMDGPU/loop_break.ll
+++ b/test/CodeGen/AMDGPU/loop_break.ll
@@ -31,7 +31,7 @@
 ; GCN: s_and_b64 vcc, exec, vcc
 ; GCN-NEXT: s_cbranch_vccnz [[FLOW:BB[0-9]+_[0-9]+]]
 
-; GCN: ; BB#2: ; %bb4
+; GCN: ; %bb.2: ; %bb4
 ; GCN: buffer_load_dword
 ; GCN: v_cmp_ge_i32_e32 vcc,
 ; GCN: s_or_b64 [[MASK]], vcc, [[INITMASK]]
@@ -41,8 +41,7 @@
 ; GCN: s_andn2_b64 exec, exec, [[MASK]]
 ; GCN-NEXT: s_cbranch_execnz [[LOOP_ENTRY]]
 
-; GCN: ; BB#4: ; %bb9
-; GCN-NEXT: s_or_b64 exec, exec, [[MASK]]
+; GCN: ; %bb.4: ; %bb9
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @break_loop(i32 %arg) #0 {
 bb:
diff --git a/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll b/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
index e1a2af6c7ef90..778467207a012 100644
--- a/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
+++ b/test/CodeGen/AMDGPU/lower-mem-intrinsics.ll
@@ -1,5 +1,4 @@
 ; RUN: opt -S -amdgpu-lower-intrinsics %s | FileCheck -check-prefix=OPT %s
-; RUN: opt -S -amdgpu-lower-intrinsics -use-wide-memcpy-loop-lowering=true %s | FileCheck -check-prefix=WOPT %s
 
 declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture readonly, i64, i32, i1) #1
 declare void @llvm.memcpy.p1i8.p3i8.i32(i8 addrspace(1)* nocapture, i8 addrspace(3)* nocapture readonly, i32, i32, i1) #1
@@ -18,21 +17,14 @@ define amdgpu_kernel void @max_size_small_static_memcpy_caller0(i8 addrspace(1)*
 ; Smallest static size which will be expanded
 ; OPT-LABEL: @min_size_large_static_memcpy_caller0(
 ; OPT-NOT: call
-; OPT: getelementptr
-; OPT-NEXT: load i8
-; OPT: getelementptr
-; OPT-NEXT: store i8
-
-; WOPT-LABEL: @min_size_large_static_memcpy_caller0(
-; WOPT-NOT: call
-; WOPT: br label %load-store-loop
-; WOPT: [[T1:%[0-9]+]] = getelementptr inbounds i8, i8 addrspace(1)* %src, i64 %loop-index
-; WOPT-NEXT: [[T2:%[0-9]+]] = load i8, i8 addrspace(1)* [[T1]]
-; WOPT-NEXT: [[T3:%[0-9]+]] = getelementptr inbounds i8, i8 addrspace(1)* %dst, i64 %loop-index
-; WOPT-NEXT: store i8 [[T2]], i8 addrspace(1)* [[T3]]
-; WOPT-NEXT: [[T4:%[0-9]+]] = add i64 %loop-index, 1
-; WOPT-NEXT: [[T5:%[0-9]+]] = icmp ult i64 [[T4]], 1025
-; WOPT-NEXT: br i1 [[T5]], label %load-store-loop, label %memcpy-split
+; OPT: br label %load-store-loop
+; OPT: [[T1:%[0-9]+]] = getelementptr inbounds i8, i8 addrspace(1)* %src, i64 %loop-index
+; OPT-NEXT: [[T2:%[0-9]+]] = load i8, i8 addrspace(1)* [[T1]]
+; OPT-NEXT: [[T3:%[0-9]+]] = getelementptr inbounds i8, i8 addrspace(1)* %dst, i64 %loop-index
+; OPT-NEXT: store i8 [[T2]], i8 addrspace(1)* [[T3]]
+; OPT-NEXT: [[T4:%[0-9]+]] = add i64 %loop-index, 1
+; OPT-NEXT: [[T5:%[0-9]+]] = icmp ult i64 [[T4]], 1025
+; OPT-NEXT: br i1 [[T5]], label %load-store-loop, label %memcpy-split
 define amdgpu_kernel void @min_size_large_static_memcpy_caller0(i8 addrspace(1)* %dst, i8 addrspace(1)* %src) #0 {
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 1025, i32 1, i1 false)
   ret void
diff --git a/test/CodeGen/AMDGPU/lshl64-to-32.ll b/test/CodeGen/AMDGPU/lshl64-to-32.ll
index 5ff6b71c1f02e..b7e5eab275c13 100644
--- a/test/CodeGen/AMDGPU/lshl64-to-32.ll
+++ b/test/CodeGen/AMDGPU/lshl64-to-32.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn < %s | FileCheck %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; CHECK-LABEL: {{^}}zext_shl64_to_32:
-; CHECK: s_lshl_b32
-; CHECK-NOT: s_lshl_b64
+; GCN-LABEL: {{^}}zext_shl64_to_32:
+; GCN: s_lshl_b32
+; GCN-NOT: s_lshl_b64
 define amdgpu_kernel void @zext_shl64_to_32(i64 addrspace(1)* nocapture %out, i32 %x) {
   %and = and i32 %x, 1073741823
   %ext = zext i32 %and to i64
@@ -11,9 +11,9 @@ define amdgpu_kernel void @zext_shl64_to_32(i64 addrspace(1)* nocapture %out, i3
   ret void
 }
 
-; CHECK-LABEL: {{^}}sext_shl64_to_32:
-; CHECK: s_lshl_b32
-; CHECK-NOT: s_lshl_b64
+; GCN-LABEL: {{^}}sext_shl64_to_32:
+; GCN: s_lshl_b32
+; GCN-NOT: s_lshl_b64
 define amdgpu_kernel void @sext_shl64_to_32(i64 addrspace(1)* nocapture %out, i32 %x) {
   %and = and i32 %x, 536870911
   %ext = sext i32 %and to i64
@@ -22,9 +22,9 @@ define amdgpu_kernel void @sext_shl64_to_32(i64 addrspace(1)* nocapture %out, i3
   ret void
 }
 
-; CHECK-LABEL: {{^}}zext_shl64_overflow:
-; CHECK: s_lshl_b64
-; CHECK-NOT: s_lshl_b32
+; GCN-LABEL: {{^}}zext_shl64_overflow:
+; GCN: s_lshl_b64
+; GCN-NOT: s_lshl_b32
 define amdgpu_kernel void @zext_shl64_overflow(i64 addrspace(1)* nocapture %out, i32 %x) {
   %and = and i32 %x, 2147483647
   %ext = zext i32 %and to i64
@@ -33,9 +33,9 @@ define amdgpu_kernel void @zext_shl64_overflow(i64 addrspace(1)* nocapture %out,
   ret void
 }
 
-; CHECK-LABEL: {{^}}sext_shl64_overflow:
-; CHECK: s_lshl_b64
-; CHECK-NOT: s_lshl_b32
+; GCN-LABEL: {{^}}sext_shl64_overflow:
+; GCN: s_lshl_b64
+; GCN-NOT: s_lshl_b32
 define amdgpu_kernel void @sext_shl64_overflow(i64 addrspace(1)* nocapture %out, i32 %x) {
   %and = and i32 %x, 2147483647
   %ext = sext i32 %and to i64
@@ -43,3 +43,37 @@ define amdgpu_kernel void @sext_shl64_overflow(i64 addrspace(1)* nocapture %out,
   store i64 %shl, i64 addrspace(1)* %out, align 4
   ret void
 }
+
+; GCN-LABEL: {{^}}mulu24_shl64:
+; GCN: v_mul_u32_u24_e32 [[M:v[0-9]+]], 7, v{{[0-9]+}}
+; GCN: v_lshlrev_b32_e32 v{{[0-9]+}}, 2, [[M]]
+define amdgpu_kernel void @mulu24_shl64(i32 addrspace(1)* nocapture %arg) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = and i32 %tmp, 6
+  %mulconv = mul nuw nsw i32 %tmp1, 7
+  %tmp2 = zext i32 %mulconv to i64
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp2
+  store i32 0, i32 addrspace(1)* %tmp3, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}muli24_shl64:
+; GCN: v_mul_i32_i24_e32 [[M:v[0-9]+]], -7, v{{[0-9]+}}
+; GCN: v_lshlrev_b32_e32 v{{[0-9]+}}, 3, [[M]]
+define amdgpu_kernel void @muli24_shl64(i64 addrspace(1)* nocapture %arg, i32 addrspace(1)* nocapture readonly %arg1) {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp2 = sext i32 %tmp to i64
+  %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp2
+  %tmp4 = load i32, i32 addrspace(1)* %tmp3, align 4
+  %tmp5 = or i32 %tmp4, -8388608
+  %tmp6 = mul nsw i32 %tmp5, -7
+  %tmp7 = zext i32 %tmp6 to i64
+  %tmp8 = shl nuw nsw i64 %tmp7, 3
+  %tmp9 = getelementptr inbounds i64, i64 addrspace(1)* %arg, i64 %tmp2
+  store i64 %tmp8, i64 addrspace(1)* %tmp9, align 8
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/test/CodeGen/AMDGPU/lshr.v2i16.ll b/test/CodeGen/AMDGPU/lshr.v2i16.ll
index 6a90a7a9f2eb3..f6b6a79ae6d21 100644
--- a/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=CIVI %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=CIVI %s
 
@@ -8,11 +8,11 @@
 ; GFX9: v_mov_b32_e32 [[VLHS:v[0-9]+]], [[LHS]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[VLHS]]
 
-; VI: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-DAG: v_lshrrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; CI: v_lshrrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
-; CIVI: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, 16
-; CIVI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; CI-DAG:   v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
+; CIVI-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, 16
+; CIVI:     v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
   %result = lshr <2 x i16> %lhs, %rhs
   store <2 x i16> %result, <2 x i16> addrspace(1)* %out
@@ -20,8 +20,8 @@ define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 }
 
 ; GCN-LABEL: {{^}}v_lshr_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 
 ; VI: v_lshrrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -52,7 +52,7 @@ define amdgpu_kernel void @v_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 
 ; GCN-LABEL: {{^}}lshr_v_s_v2i16:
 ; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -67,7 +67,7 @@ define amdgpu_kernel void @lshr_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
 
 ; GCN-LABEL: {{^}}lshr_s_v_v2i16:
 ; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -81,7 +81,7 @@ define amdgpu_kernel void @lshr_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16
 }
 
 ; GCN-LABEL: {{^}}lshr_imm_v_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], [[RHS]], 8
 define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -95,7 +95,7 @@ define amdgpu_kernel void @lshr_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 }
 
 ; GCN-LABEL: {{^}}lshr_v_imm_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_lshrrev_b16 [[RESULT:v[0-9]+]], 8, [[LHS]]
 define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -109,11 +109,11 @@ define amdgpu_kernel void @lshr_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i
 }
 
 ; GCN-LABEL: {{^}}v_lshr_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -128,10 +128,10 @@ define amdgpu_kernel void @v_lshr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16>
 }
 
 ; GCN-LABEL: {{^}}lshr_v_imm_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_lshrrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @lshr_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/macro-fusion-cluster-vcc-uses.mir b/test/CodeGen/AMDGPU/macro-fusion-cluster-vcc-uses.mir
index 768acf35eeae3..6c6b19a04c622 100644
--- a/test/CodeGen/AMDGPU/macro-fusion-cluster-vcc-uses.mir
+++ b/test/CodeGen/AMDGPU/macro-fusion-cluster-vcc-uses.mir
@@ -1,17 +1,17 @@
-# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass machine-scheduler -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -run-pass machine-scheduler -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: name: cluster_add_addc
 # GCN: S_NOP 0, implicit-def %vcc
-# GCN: dead %2, %3 = V_ADD_I32_e64 %0, %1, implicit %exec
-# GCN: dead %4, dead %5 = V_ADDC_U32_e64 %6, %7, %3, implicit %exec
+# GCN: dead %2:vgpr_32, %3:sreg_64_xexec = V_ADD_I32_e64 %0, %1, implicit %exec
+# GCN: dead %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %6, %7, %3, implicit %exec
 name: cluster_add_addc
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
@@ -27,10 +27,10 @@ body: |
 ...
 
 # GCN-LABEL: name: interleave_add64s
-# GCN: dead %8, %9 = V_ADD_I32_e64 %0, %1, implicit %exec
-# GCN-NEXT: dead %12, dead %13 = V_ADDC_U32_e64 %4, %5, %9, implicit %exec
-# GCN-NEXT: dead %10, %11 = V_ADD_I32_e64 %2, %3, implicit %exec
-# GCN-NEXT: dead %14, dead %15 = V_ADDC_U32_e64 %6, %7, %11, implicit %exec
+# GCN: dead %8:vgpr_32, %9:sreg_64_xexec = V_ADD_I32_e64 %0, %1, implicit %exec
+# GCN-NEXT: dead %12:vgpr_32, dead %13:sreg_64_xexec = V_ADDC_U32_e64 %4, %5, %9, implicit %exec
+# GCN-NEXT: dead %10:vgpr_32, %11:sreg_64_xexec = V_ADD_I32_e64 %2, %3, implicit %exec
+# GCN-NEXT: dead %14:vgpr_32, dead %15:sreg_64_xexec = V_ADDC_U32_e64 %6, %7, %11, implicit %exec
 name: interleave_add64s
 registers:
   - { id: 0, class: vgpr_32 }
@@ -42,13 +42,13 @@ registers:
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
   - { id: 8, class: vgpr_32 }
-  - { id: 9, class: sreg_64 }
+  - { id: 9, class: sreg_64_xexec }
   - { id: 10, class: vgpr_32 }
-  - { id: 11, class: sreg_64 }
+  - { id: 11, class: sreg_64_xexec }
   - { id: 12, class: vgpr_32 }
-  - { id: 13, class: sreg_64 }
+  - { id: 13, class: sreg_64_xexec }
   - { id: 14, class: vgpr_32 }
-  - { id: 15, class: sreg_64 }
+  - { id: 15, class: sreg_64_xexec }
 
 body: |
   bb.0:
@@ -71,15 +71,15 @@ body: |
 
 # GCN-LABEL: name: cluster_mov_addc
 # GCN: S_NOP 0, implicit-def %vcc
-# GCN-NEXT: %2 = S_MOV_B64 0
-# GCN-NEXT: dead %3, dead %4 = V_ADDC_U32_e64 %0, %1, %2, implicit %exec
+# GCN-NEXT: %2:sreg_64_xexec = S_MOV_B64 0
+# GCN-NEXT: dead %3:vgpr_32, dead %4:sreg_64_xexec = V_ADDC_U32_e64 %0, %1, %2, implicit %exec
 name: cluster_mov_addc
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
-  - { id: 2, class: sreg_64 }
+  - { id: 2, class: sreg_64_xexec }
   - { id: 3, class: vgpr_32 }
-  - { id: 4, class: sreg_64 }
+  - { id: 4, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
@@ -93,23 +93,23 @@ body: |
 ...
 
 # GCN-LABEL: name: no_cluster_add_addc_diff_sgpr
-# GCN: dead %2, dead %3 = V_ADD_I32_e64 %0, %1, implicit %exec
-# GCN-NEXT: %6 = V_MOV_B32_e32 0, implicit %exec
-# GCN-NEXT: %7 = V_MOV_B32_e32 0, implicit %exec
+# GCN: dead %2:vgpr_32, dead %3:sreg_64_xexec = V_ADD_I32_e64 %0, %1, implicit %exec
+# GCN-NEXT: %6:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# GCN-NEXT: %7:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
 # GCN-NEXT: S_NOP 0, implicit-def %vcc
-# GCN-NEXT: %8 = S_MOV_B64 0
-# GCN-NEXT: dead %4, dead %5 = V_ADDC_U32_e64 %6, %7, %8, implicit %exec
+# GCN-NEXT: %8:sreg_64_xexec = S_MOV_B64 0
+# GCN-NEXT: dead %4:vgpr_32, dead %5:sreg_64_xexec = V_ADDC_U32_e64 %6, %7, %8, implicit %exec
 name: no_cluster_add_addc_diff_sgpr
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
-  - { id: 8, class: sreg_64 }
+  - { id: 8, class: sreg_64_xexec }
 body: |
   bb.0:
     %0 = V_MOV_B32_e32 0, implicit %exec
@@ -123,16 +123,16 @@ body: |
 ...
 # GCN-LABEL: name: cluster_sub_subb
 # GCN: S_NOP 0, implicit-def %vcc
-# GCN: dead %2, %3 = V_SUB_I32_e64 %0, %1, implicit %exec
-# GCN: dead %4, dead %5 = V_SUBB_U32_e64 %6, %7, %3, implicit %exec
+# GCN: dead %2:vgpr_32, %3:sreg_64_xexec = V_SUB_I32_e64 %0, %1, implicit %exec
+# GCN: dead %4:vgpr_32, dead %5:sreg_64_xexec = V_SUBB_U32_e64 %6, %7, %3, implicit %exec
 name: cluster_sub_subb
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
@@ -149,16 +149,16 @@ body: |
 
 # GCN-LABEL: name: cluster_cmp_cndmask
 # GCN: S_NOP 0, implicit-def %vcc
-# GCN-NEXT: %3 = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
-# GCN-NEXT: dead %4 = V_CNDMASK_B32_e64 %0, %1, %3, implicit %exec
+# GCN-NEXT: %3:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
+# GCN-NEXT: dead %4:vgpr_32 = V_CNDMASK_B32_e64 %0, %1, %3, implicit %exec
 name: cluster_cmp_cndmask
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
 
@@ -172,16 +172,16 @@ body: |
 ...
 
 # GCN-LABEL: name: cluster_multi_use_cmp_cndmask
-# GCN: %4 = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
-# GCN-NEXT: dead %5 = V_CNDMASK_B32_e64 %2, %1, %4, implicit %exec
-# GCN-NEXT: dead %6 = V_CNDMASK_B32_e64 %1, %3, %4, implicit %exec
+# GCN: %4:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
+# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 %2, %1, %4, implicit %exec
+# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 %1, %3, %4, implicit %exec
 name: cluster_multi_use_cmp_cndmask
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
   - { id: 3, class: vgpr_32 }
-  - { id: 4, class: sreg_64 }
+  - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: vgpr_32 }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
@@ -200,17 +200,17 @@ body: |
 ...
 
 # GCN-LABEL: name: cluster_multi_use_cmp_cndmask2
-# GCN: %4 = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
-# GCN-NEXT: dead %5 = V_CNDMASK_B32_e64 %2, %1, %4, implicit %exec
-# GCN-NEXT: %3 = V_MOV_B32_e32 0, implicit %exec
-# GCN-NEXT: dead %6 = V_CNDMASK_B32_e64 %1, %3, %4, implicit %exec
+# GCN: %4:sreg_64_xexec = V_CMP_EQ_I32_e64 %0, %1, implicit %exec
+# GCN-NEXT: dead %5:vgpr_32 = V_CNDMASK_B32_e64 %2, %1, %4, implicit %exec
+# GCN-NEXT: %3:vgpr_32 = V_MOV_B32_e32 0, implicit %exec
+# GCN-NEXT: dead %6:vgpr_32 = V_CNDMASK_B32_e64 %1, %3, %4, implicit %exec
 name: cluster_multi_use_cmp_cndmask2
 registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
   - { id: 3, class: vgpr_32 }
-  - { id: 4, class: sreg_64 }
+  - { id: 4, class: sreg_64_xexec }
   - { id: 5, class: vgpr_32 }
   - { id: 6, class: vgpr_32 }
   - { id: 7, class: vgpr_32 }
diff --git a/test/CodeGen/AMDGPU/mad-combine.ll b/test/CodeGen/AMDGPU/mad-combine.ll
index 8a6bf853a7c6a..0b360f6ecefb5 100644
--- a/test/CodeGen/AMDGPU/mad-combine.ll
+++ b/test/CodeGen/AMDGPU/mad-combine.ll
@@ -1,12 +1,12 @@
 ; Make sure we still form mad even when unsafe math or fp-contract is allowed instead of fma.
 
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD  -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-SAFE -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-STD -check-prefix=SI-STD-UNSAFE -check-prefix=FUNC %s
 
 ; Make sure we don't form mad with denormals
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-FASTFMAF -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=verde -mattr=+fp32-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=SI-DENORM -check-prefix=SI-DENORM-SLOWFMAF -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare float @llvm.fabs.f32(float) #0
@@ -231,6 +231,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_1_f32(float addrspace(1)* noalias
 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
+; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
 
 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], [[C]]
 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], -[[A]], [[B]], [[D]]
@@ -305,6 +306,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32(float addrspace(1)* noalias
 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
+; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
 
 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], [[A]], -[[B]], -[[C]]
 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], -[[B]], -[[D]]
@@ -348,6 +350,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_neg(float addrspace(1
 ; SI-DAG: buffer_load_dword [[A:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
+; SI-DAG: buffer_load_dword [[D:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
 
 ; SI-STD-DAG: v_mad_f32 [[RESULT0:v[0-9]+]], -[[A]], [[B]], -[[C]]
 ; SI-STD-DAG: v_mad_f32 [[RESULT1:v[0-9]+]], [[A]], [[B]], -[[D]]
@@ -401,7 +404,7 @@ define amdgpu_kernel void @combine_to_mad_fsub_2_f32_2uses_mul(float addrspace(1
 
 ; SI-DENORM: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[D]], [[E]]
 ; SI-DENORM: v_fma_f32 [[TMP1:v[0-9]+]], [[A]], [[B]], [[TMP0]]
-; SI-DENORM: v_sub_f32_e32 [[RESULT1:v[0-9]+]], [[TMP1]], [[C]]
+; SI-DENORM: v_sub_f32_e32 [[RESULT:v[0-9]+]], [[TMP1]], [[C]]
 
 ; SI: buffer_store_dword [[RESULT]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
 define amdgpu_kernel void @aggressive_combine_to_mad_fsub_0_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) #1 {
diff --git a/test/CodeGen/AMDGPU/mad-mix-hi.ll b/test/CodeGen/AMDGPU/mad-mix-hi.ll
new file mode 100644
index 0000000000000..17bade282b158
--- /dev/null
+++ b/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -0,0 +1,145 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %vec.result = insertelement <2 x half> undef, half %cvt.result, i32 1
+  ret <2 x half> %vec.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v3, 0x3c00
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %vec.result = insertelement <2 x half> <half 1.0, half undef>, half %cvt.result, i32 1
+  ret <2 x half> %vec.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src1, half %src2, half %lo) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %vec = insertelement <2 x half> undef, half %lo, i32 0
+  %vec.result = insertelement <2 x half> %vec, half %cvt.result, i32 1
+  ret <2 x half> %vec.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; GFX9: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %bc = bitcast half %cvt.result to i16
+  %ext = zext i16 %bc to i32
+  %shr = shl i32 %ext, 16
+  ret i32 %shr
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; GFX9: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %bc = bitcast half %cvt.result to i16
+  %ext = sext i16 %bc to i32
+  %shr = shl i32 %ext, 16
+  ret i32 %shr
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp{{$}}
+; GFX9: v_cvt_f16_f32_e32 v0, v0
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_precvt(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %max = call float @llvm.maxnum.f32(float %result, float 0.0)
+  %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+  %cvt.result = fptrunc float %clamp to half
+  %vec.result = insertelement <2 x half> undef, half %cvt.result, i32 1
+  ret <2 x half> %vec.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp{{$}}
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
+  %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
+  %vec.result = insertelement <2 x half> undef, half %clamp, i32 1
+  ret <2 x half> %vec.result
+}
+
+
+; GCN-LABEL: {{^}}v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]{{$}}
+; GFX9-NEXT: global_store_short v{{\[[0-9]+:[0-9]+\]}}, v3
+; GFX9-NEXT: v_mad_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp{{$}}
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  store volatile half %cvt.result, half addrspace(1)* undef
+  %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
+  %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
+  %vec.result = insertelement <2 x half> undef, half %clamp, i32 1
+  ret <2 x half> %vec.result
+}
+
+declare half @llvm.minnum.f16(half, half) #1
+declare half @llvm.maxnum.f16(half, half) #1
+declare float @llvm.minnum.f32(float, float) #1
+declare float @llvm.maxnum.f32(float, float) #1
+declare float @llvm.fmuladd.f32(float, float, float) #1
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/mad-mix-lo.ll b/test/CodeGen/AMDGPU/mad-mix-lo.ll
new file mode 100644
index 0000000000000..fc0745e8ce34f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -0,0 +1,307 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
+
+; GCN-LABEL: mixlo_simple:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32_e32
+; CIVI: v_cvt_f16_f32_e32
+define half @mixlo_simple(float %src0, float %src1, float %src2) #0 {
+  %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
+  %cvt.result = fptrunc float %result to half
+  ret half %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f16lo:
+; GFX9: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]{{$}}
+; CI: v_mac_f32
+; CIVI: v_cvt_f16_f32
+define half @v_mad_mixlo_f16_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %cvt.result = fptrunc float %result to half
+  ret half %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0]{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32
+define half @v_mad_mixlo_f16_f16lo_f16lo_f32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  %cvt.result = fptrunc float %result to half
+  ret half %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp{{$}}
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32_e32 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]$}}
+define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_post_cvt(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  %cvt.result = fptrunc float %result to half
+  %max = call half @llvm.maxnum.f16(half %cvt.result, half 0.0)
+  %clamp = call half @llvm.minnum.f16(half %max, half 1.0)
+  ret half %clamp
+}
+
+; GCN-LABEL: {{^}}v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp{{$}}
+; GFX9-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32_e64 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
+define half @v_mad_mixlo_f16_f16lo_f16lo_f32_clamp_pre_cvt(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  %max = call float @llvm.maxnum.f32(float %result, float 0.0)
+  %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+  %cvt.result = fptrunc float %clamp to half
+  ret half %cvt.result
+}
+
+; FIXME: Should abe able to avoid extra register because first
+; operation only clobbers relevant lane.
+; GCN-LABEL: {{^}}v_mad_mix_v2f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]{{$}}
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]{{$}}
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  %cvt.result = fptrunc <2 x float> %result to <2 x half>
+  ret <2 x half> %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v3f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v3, v6 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixlo_f16 v1, v1, v4, v7 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixlo_f16 v2, v2, v5, v8 op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
+  %src0.ext = fpext <3 x half> %src0 to <3 x float>
+  %src1.ext = fpext <3 x half> %src1 to <3 x float>
+  %src2.ext = fpext <3 x half> %src2 to <3 x float>
+  %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
+  %cvt.result = fptrunc <3 x float> %result to <3 x half>
+  ret <3 x half> %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v4f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_setpc_b64
+define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
+  %src0.ext = fpext <4 x half> %src0 to <4 x float>
+  %src1.ext = fpext <4 x half> %src1 to <4 x float>
+  %src2.ext = fpext <4 x half> %src2 to <4 x float>
+  %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
+  %cvt.result = fptrunc <4 x float> %result to <4 x half>
+  ret <4 x half> %cvt.result
+}
+
+; FIXME: Fold clamp
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt:
+; GFX9: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp{{$}}
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp{{$}}
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  %cvt.result = fptrunc <2 x float> %result to <2 x half>
+  %max = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %cvt.result, <2 x half> zeroinitializer)
+  %clamp = call <2 x half> @llvm.minnum.v2f16(<2 x half> %max, <2 x half> <half 1.0, half 1.0>)
+  ret <2 x half> %clamp
+}
+
+; FIXME: Should be packed into 2 registers per argument?
+; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_postcvt:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v2, v2, v5, v8 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v0, v0, v0 clamp
+; GFX9-NEXT: v_mad_mixlo_f16 v0, v0, v3, v6 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v0, v1, v4, v7 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX9-NEXT: s_setpc_b64
+define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
+  %src0.ext = fpext <3 x half> %src0 to <3 x float>
+  %src1.ext = fpext <3 x half> %src1 to <3 x float>
+  %src2.ext = fpext <3 x half> %src2 to <3 x float>
+  %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
+  %cvt.result = fptrunc <3 x float> %result to <3 x half>
+  %max = call <3 x half> @llvm.maxnum.v3f16(<3 x half> %cvt.result, <3 x half> zeroinitializer)
+  %clamp = call <3 x half> @llvm.minnum.v3f16(<3 x half> %max, <3 x half> <half 1.0, half 1.0, half 1.0>)
+  ret <3 x half> %clamp
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v4f32_clamp_postcvt:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v6, v0, v2, v4 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixlo_f16 v2, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9-DAG: v_mov_b32_e32 v0, v6
+; GFX9-DAG: v_mov_b32_e32 v1, v2
+; GFX9: s_setpc_b64
+define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
+  %src0.ext = fpext <4 x half> %src0 to <4 x float>
+  %src1.ext = fpext <4 x half> %src1 to <4 x float>
+  %src2.ext = fpext <4 x half> %src2 to <4 x float>
+  %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
+  %cvt.result = fptrunc <4 x float> %result to <4 x half>
+  %max = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %cvt.result, <4 x half> zeroinitializer)
+  %clamp = call <4 x half> @llvm.minnum.v4f16(<4 x half> %max, <4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>)
+  ret <4 x half> %clamp
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt_lo:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  %cvt.result = fptrunc <2 x float> %result to <2 x half>
+  %cvt.lo = extractelement <2 x half> %cvt.result, i32 0
+  %max.lo = call half @llvm.maxnum.f16(half %cvt.lo, half 0.0)
+  %clamp.lo = call half @llvm.minnum.f16(half %max.lo, half 1.0)
+  %insert = insertelement <2 x half> %cvt.result, half %clamp.lo, i32 0
+  ret <2 x half> %insert
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_postcvt_hi:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: s_setpc_b64
+define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  %cvt.result = fptrunc <2 x float> %result to <2 x half>
+  %cvt.hi = extractelement <2 x half> %cvt.result, i32 1
+  %max.hi = call half @llvm.maxnum.f16(half %cvt.hi, half 0.0)
+  %clamp.hi = call half @llvm.minnum.f16(half %max.hi, half 1.0)
+  %insert = insertelement <2 x half> %cvt.result, half %clamp.hi, i32 1
+  ret <2 x half> %insert
+}
+
+; FIXME: Should be able to use mixlo/mixhi
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_clamp_precvt:
+; GFX9: v_mad_mix_f32 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9: v_cvt_f16_f32_e32 v1, v3
+; GFX9: v_cvt_f16_f32_e32 v0, v0
+; GFX9: v_and_b32_e32 v1, 0xffff, v1
+; GFX9: v_lshl_or_b32 v0, v0, 16, v1
+; GFX9: s_setpc_b64
+define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  %max = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %result, <2 x float> zeroinitializer)
+  %clamp = call <2 x float> @llvm.minnum.v2f32(<2 x float> %max, <2 x float> <float 1.0, float 1.0>)
+  %cvt.result = fptrunc <2 x float> %clamp to <2 x half>
+  ret <2 x half> %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v3f32_clamp_precvt:
+; GFX9: v_mad_mix_f32 v0, v0, v3, v6 op_sel_hi:[1,1,1] clamp
+; GFX9: v_mad_mix_f32 v1, v1, v4, v7 op_sel_hi:[1,1,1] clamp
+; GFX9: v_mad_mix_f32 v2, v2, v5, v8 op_sel_hi:[1,1,1] clamp
+; GFX9: v_cvt_f16_f32
+; GFX9: v_cvt_f16_f32
+; GFX9: v_cvt_f16_f32
+define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %src1, <3 x half> %src2) #0 {
+  %src0.ext = fpext <3 x half> %src0 to <3 x float>
+  %src1.ext = fpext <3 x half> %src1 to <3 x float>
+  %src2.ext = fpext <3 x half> %src2 to <3 x float>
+  %result = tail call <3 x float> @llvm.fmuladd.v3f32(<3 x float> %src0.ext, <3 x float> %src1.ext, <3 x float> %src2.ext)
+  %max = call <3 x float> @llvm.maxnum.v3f32(<3 x float> %result, <3 x float> zeroinitializer)
+  %clamp = call <3 x float> @llvm.minnum.v3f32(<3 x float> %max, <3 x float> <float 1.0, float 1.0, float 1.0>)
+  %cvt.result = fptrunc <3 x float> %clamp to <3 x half>
+  ret <3 x half> %cvt.result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v4f32_clamp_precvt:
+; GFX9: v_mad_mix_f32 v6, v0, v2, v4 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9: v_mad_mix_f32 v0, v0, v2, v4 op_sel_hi:[1,1,1] clamp
+; GFX9: v_mad_mix_f32 v2, v1, v3, v5 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GFX9: v_mad_mix_f32 v1, v1, v3, v5 op_sel_hi:[1,1,1] clamp
+; GFX9: v_cvt_f16_f32
+; GFX9: v_cvt_f16_f32
+; GFX9: v_cvt_f16_f32
+; GFX9: v_cvt_f16_f32
+define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %src1, <4 x half> %src2) #0 {
+  %src0.ext = fpext <4 x half> %src0 to <4 x float>
+  %src1.ext = fpext <4 x half> %src1 to <4 x float>
+  %src2.ext = fpext <4 x half> %src2 to <4 x float>
+  %result = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %src0.ext, <4 x float> %src1.ext, <4 x float> %src2.ext)
+  %max = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %result, <4 x float> zeroinitializer)
+  %clamp = call <4 x float> @llvm.minnum.v4f32(<4 x float> %max, <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>)
+  %cvt.result = fptrunc <4 x float> %clamp to <4 x half>
+  ret <4 x half> %cvt.result
+}
+
+declare half @llvm.minnum.f16(half, half) #1
+declare <2 x half> @llvm.minnum.v2f16(<2 x half>, <2 x half>) #1
+declare <3 x half> @llvm.minnum.v3f16(<3 x half>, <3 x half>) #1
+declare <4 x half> @llvm.minnum.v4f16(<4 x half>, <4 x half>) #1
+
+declare half @llvm.maxnum.f16(half, half) #1
+declare <2 x half> @llvm.maxnum.v2f16(<2 x half>, <2 x half>) #1
+declare <3 x half> @llvm.maxnum.v3f16(<3 x half>, <3 x half>) #1
+declare <4 x half> @llvm.maxnum.v4f16(<4 x half>, <4 x half>) #1
+
+declare float @llvm.minnum.f32(float, float) #1
+declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>) #1
+declare <3 x float> @llvm.minnum.v3f32(<3 x float>, <3 x float>) #1
+declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>) #1
+
+declare float @llvm.maxnum.f32(float, float) #1
+declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>) #1
+declare <3 x float> @llvm.maxnum.v3f32(<3 x float>, <3 x float>) #1
+declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>) #1
+
+declare float @llvm.fmuladd.f32(float, float, float) #1
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+declare <3 x float> @llvm.fmuladd.v3f32(<3 x float>, <3 x float>, <3 x float>) #1
+declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/mad-mix.ll b/test/CodeGen/AMDGPU/mad-mix.ll
new file mode 100644
index 0000000000000..c95b7aa5d018c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/mad-mix.ll
@@ -0,0 +1,509 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI,CI %s
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] ; encoding: [0x00,0x40,0xa0,0xd3,0x00,0x03,0x0a,0x1c]
+; VI: v_mac_f32
+; CI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_int:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] ; encoding
+; CIVI: v_mac_f32
+define float @v_mad_mix_f32_f16hi_f16hi_f16hi_int(i32 %src0, i32 %src1, i32 %src2) #0 {
+  %src0.hi = lshr i32 %src0, 16
+  %src1.hi = lshr i32 %src1, 16
+  %src2.hi = lshr i32 %src2, 16
+  %src0.i16 = trunc i32 %src0.hi to i16
+  %src1.i16 = trunc i32 %src1.hi to i16
+  %src2.i16 = trunc i32 %src2.hi to i16
+  %src0.fp16 = bitcast i16 %src0.i16 to half
+  %src1.fp16 = bitcast i16 %src1.i16 to half
+  %src2.fp16 = bitcast i16 %src2.i16 to half
+  %src0.ext = fpext half %src0.fp16 to float
+  %src1.ext = fpext half %src1.fp16 to float
+  %src2.ext = fpext half %src2.fp16 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] ; encoding
+; VI: v_mac_f32
+; CI: v_mad_f32
+define float @v_mad_mix_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.hi = extractelement <2 x half> %src0, i32 1
+  %src1.hi = extractelement <2 x half> %src1, i32 1
+  %src2.hi = extractelement <2 x half> %src2, i32 1
+  %src0.ext = fpext half %src0.hi to float
+  %src1.ext = fpext half %src1.hi to float
+  %src2.ext = fpext half %src2.hi to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32:
+; GFX9: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2 op_sel_hi:[1,1,1]
+
+; CIVI: v_mac_f32
+define <2 x float> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2.ext = fpext <2 x half> %src2 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_shuffle:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mad_mix_f32 v1, v0, v3, v2 op_sel:[0,1,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v3, v2 op_sel:[1,0,1] op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mac_f32
+define <2 x float> @v_mad_mix_v2f32_shuffle(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.shuf = shufflevector <2 x half> %src0, <2 x half> undef, <2 x i32> <i32 1, i32 0>
+  %src1.shuf = shufflevector <2 x half> %src1, <2 x half> undef, <2 x i32> <i32 0, i32 1>
+  %src2.shuf = shufflevector <2 x half> %src2, <2 x half> undef, <2 x i32> <i32 1, i32 1>
+  %src0.ext = fpext <2 x half> %src0.shuf to <2 x float>
+  %src1.ext = fpext <2 x half> %src1.shuf to <2 x float>
+  %src2.ext = fpext <2 x half> %src2.shuf to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2.ext)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_negf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel_hi:[1,1,1] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.neg = fsub float -0.0, %src0.ext
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_absf16lo_f16lo_f16lo:
+; GFX9: v_mad_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_absf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_negabsf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -|v0|, v1, v2 op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negabsf16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %src0.ext.neg.abs = fsub float -0.0, %src0.ext.abs
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.neg.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -v2 op_sel_hi:[1,1,0] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_negf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.neg = fsub float -0.0, %src2
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_absf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, |v2| op_sel_hi:[1,1,0] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_absf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.abs = call float @llvm.fabs.f32(float %src2)
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.abs)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_negabsf32:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, -|v2| op_sel_hi:[1,1,0] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_f16lo_f16lo_negabsf32(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.abs = call float @llvm.fabs.f32(float %src2)
+  %src2.neg.abs = fsub float -0.0, %src2.abs
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.neg.abs)
+  ret float %result
+}
+
+; TODO: Fold inline immediates. Need to be careful because it is an
+; f16 inline immediate that may be converted to f32, not an actual f32
+; inline immediate.
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imm1:
+; GCN: s_waitcnt
+; GFX9: v_mov_b32_e32 v2, 1.0
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+
+; CIVI: v_mad_f32 v0, v0, v1, 1.0
+; GCN-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f32imm1(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 1.0)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32imminv2pi:
+; GCN: s_waitcnt
+; GFX9: v_mov_b32_e32 v2, 0.15915494
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+; VI: v_mad_f32 v0, v0, v1, 0.15915494
+define float @v_mad_mix_f32_f16lo_f16lo_f32imminv2pi(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float 0x3FC45F3060000000)
+  ret float %result
+}
+
+; Attempt to break inline immediate folding. If the operand is
+; interpreted as f32, the inline immediate is really the f16 inline
+; imm value converted to f32.
+;	fpext f16 1/2pi = 0x3e230000
+;	      f32 1/2pi = 0x3e22f983
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi:
+; GFX9: v_mov_b32_e32 v2, 0x3e230000
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+
+; CIVI: v_madak_f32 v0, v0, v1, 0x3e230000
+define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imminv2pi(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2 = fpext half 0xH3118 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_cvtf16imm63:
+; GFX9: v_mov_b32_e32 v2, 0x367c0000
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+
+; CIVI: v_madak_f32 v0, v0, v1, 0x367c0000
+define float @v_mad_mix_f32_f16lo_f16lo_cvtf16imm63(half %src0, half %src1) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2 = fpext half 0xH003F to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imm1:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 1.0
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0] ; encoding
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0] ; encoding
+define <2 x float> @v_mad_mix_v2f32_f32imm1(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 1.0, float 1.0>)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_cvtf16imminv2pi:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 0x3e230000
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0] ; encoding
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0] ; encoding
+define <2 x float> @v_mad_mix_v2f32_cvtf16imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> %src2)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_v2f32_f32imminv2pi:
+; GFX9: v_mov_b32_e32 v2, v1
+; GFX9: v_mov_b32_e32 v3, 0.15915494
+; GFX9: v_mad_mix_f32 v1, v0, v2, v3 op_sel:[1,1,0] op_sel_hi:[1,1,0] ; encoding
+; GFX9: v_mad_mix_f32 v0, v0, v2, v3 op_sel_hi:[1,1,0] ; encoding
+define <2 x float> @v_mad_mix_v2f32_f32imminv2pi(<2 x half> %src0, <2 x half> %src1) #0 {
+  %src0.ext = fpext <2 x half> %src0 to <2 x float>
+  %src1.ext = fpext <2 x half> %src1 to <2 x float>
+  %src2 = fpext <2 x half> <half 0xH3118, half 0xH3118> to <2 x float>
+  %result = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> %src0.ext, <2 x float> %src1.ext, <2 x float> <float 0x3FC45F3060000000, float 0x3FC45F3060000000>)
+  ret <2 x float> %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
+; GFX9: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp ; encoding
+; VI: v_mac_f32_e64 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
+; CI: v_mad_f32 v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}}, v{{[0-9]}} clamp{{$}}
+define float @v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %src1, <2 x half> %src2) #0 {
+  %src0.hi = extractelement <2 x half> %src0, i32 1
+  %src1.hi = extractelement <2 x half> %src1, i32 1
+  %src2.hi = extractelement <2 x half> %src2, i32 1
+  %src0.ext = fpext half %src0.hi to float
+  %src1.ext = fpext half %src1.hi to float
+  %src2.ext = fpext half %src2.hi to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  %max = call float @llvm.maxnum.f32(float %result, float 0.0)
+  %clamp = call float @llvm.minnum.f32(float %max, float 1.0)
+  ret float %clamp
+}
+
+; GCN-LABEL: no_mix_simple:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_f32 v0, v0, v1, v2
+; GCN-NEXT: s_setpc_b64
+define float @no_mix_simple(float %src0, float %src1, float %src2) #0 {
+  %result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: no_mix_simple_fabs:
+; GCN: s_waitcnt
+; GCN-NEXT: v_mad_f32 v0, |v0|, v1, v2
+; GCN-NEXT: s_setpc_b64
+define float @no_mix_simple_fabs(float %src0, float %src1, float %src2) #0 {
+  %src0.fabs = call float @llvm.fabs.f32(float %src0)
+  %result = call float @llvm.fmuladd.f32(float %src0.fabs, float %src1, float %src2)
+  ret float %result
+}
+
+; FIXME: Should abe able to select in thits case
+; All sources are converted from f16, so it doesn't matter
+; v_mad_mix_f32 flushes.
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_fma_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals(half %src0, half %src1, half %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_fma_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals(half %src0, half %src1, float %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_mul_f32
+; GFX9: v_add_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, half %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2.ext
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd:
+; GFX9: v_cvt_f32_f16
+; GFX9: v_cvt_f32_f16
+; GFX9: v_mul_f32
+; GFX9: v_add_f32
+define float @v_mad_mix_f32_f16lo_f16lo_f32_denormals_fmulfadd(half %src0, half %src1, float %src2) #1 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,1] ; encoding
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, half %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2.ext
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd:
+; GCN: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel_hi:[1,1,0] ; encoding
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_f16lo_f16lo_f32_flush_fmulfadd(half %src0, half %src1, float %src2) #0 {
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %mul = fmul float %src0.ext, %src1.ext
+  %result = fadd float %mul, %src2
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel_hi:[1,1,1] ; encoding
+; GFX9-NEXT: s_setpc_b64
+
+; CIVI: v_mad_f32
+define float @v_mad_mix_f32_negprecvtf16lo_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 0
+  %src0.neg = fsub half -0.0, %src0
+  %src0.ext = fpext half %src0.neg to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+;  %src0.ext.neg = fsub float -0.0, %src0.ext
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; Make sure we don't fold pre-cvt fneg if we already have a fabs
+; GCN-LABEL: {{^}}v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; GFX9: s_waitcnt
+define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 1
+  %src0.neg = fsub half -0.0, %src0
+  %src0.ext = fpext half %src0.neg to float
+  %src0.ext.abs = call float @llvm.fabs.f32(float %src0.ext)
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext.abs, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_precvtabsf16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, |v0|, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_precvtabsf16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %src0 = extractelement <2 x half> %src0.arg.bc, i32 1
+  %src0.abs = call half @llvm.fabs.f16(half %src0)
+  %src0.ext = fpext half %src0.abs to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fneg = fsub <2 x half> <half -0.0, half -0.0>, %src0.arg.bc
+  %src0 = extractelement <2 x half> %fneg, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; FIXME: Should be able to fold
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX9-NEXT: v_mad_mix_f32 v0, v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+  %src0 = extractelement <2 x half> %fabs, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+; FIXME: Should be able to fold
+; GCN-LABEL: {{^}}v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; GFX9: s_waitcnt
+; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0
+; GFX9-NEXT: v_mad_mix_f32 v0, -v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; GFX9-NEXT: s_setpc_b64
+define float @v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half %src1, half %src2) #0 {
+  %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+  %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+  %fneg.fabs = fsub <2 x half> <half -0.0, half -0.0>, %fabs
+  %src0 = extractelement <2 x half> %fneg.fabs, i32 1
+  %src0.ext = fpext half %src0 to float
+  %src1.ext = fpext half %src1 to float
+  %src2.ext = fpext half %src2 to float
+  %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+  ret float %result
+}
+
+declare half @llvm.fabs.f16(half) #2
+declare <2 x half> @llvm.fabs.v2f16(<2 x half>) #2
+declare float @llvm.fabs.f32(float) #2
+declare float @llvm.minnum.f32(float, float) #2
+declare float @llvm.maxnum.f32(float, float) #2
+declare float @llvm.fmuladd.f32(float, float, float) #2
+declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #2
+
+attributes #0 = { nounwind "target-features"="-fp32-denormals" }
+attributes #1 = { nounwind "target-features"="+fp32-denormals" }
+attributes #2 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/mad_64_32.ll b/test/CodeGen/AMDGPU/mad_64_32.ll
new file mode 100644
index 0000000000000..91a53eed7d750
--- /dev/null
+++ b/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -0,0 +1,169 @@
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s
+
+; GCN-LABEL: {{^}}mad_i64_i32_sextops:
+; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
+
+; SI: v_mul_lo_i32
+; SI: v_mul_hi_i32
+; SI: v_add_i32
+; SI: v_addc_u32
+define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
+  %sext0 = sext i32 %arg0 to i64
+  %sext1 = sext i32 %arg1 to i64
+  %mul = mul i64 %sext0, %sext1
+  %mad = add i64 %mul, %arg2
+  ret i64 %mad
+}
+
+; GCN-LABEL: {{^}}mad_i64_i32_sextops_commute:
+; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
+
+; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_hi_i32
+; SI: v_add_i32
+; SI: v_addc_u32
+define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
+  %sext0 = sext i32 %arg0 to i64
+  %sext1 = sext i32 %arg1 to i64
+  %mul = mul i64 %sext0, %sext1
+  %mad = add i64 %arg2, %mul
+  ret i64 %mad
+}
+
+; GCN-LABEL: {{^}}mad_u64_u32_zextops:
+; CI: v_mad_u64_u32 v[0:1], s[6:7], v0, v1, v[2:3]
+
+; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_hi_u32
+; SI: v_add_i32
+; SI: v_addc_u32
+define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
+  %sext0 = zext i32 %arg0 to i64
+  %sext1 = zext i32 %arg1 to i64
+  %mul = mul i64 %sext0, %sext1
+  %mad = add i64 %mul, %arg2
+  ret i64 %mad
+}
+
+; GCN-LABEL: {{^}}mad_u64_u32_zextops_commute:
+; CI: v_mad_u64_u32 v[0:1], s[6:7], v0, v1, v[2:3]
+
+; SI-DAG: v_mul_lo_i32
+; SI-DAG: v_mul_hi_u32
+; SI: v_add_i32
+; SI: v_addc_u32
+define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
+  %sext0 = zext i32 %arg0 to i64
+  %sext1 = zext i32 %arg1 to i64
+  %mul = mul i64 %sext0, %sext1
+  %mad = add i64 %arg2, %mul
+  ret i64 %mad
+}
+
+
+
+
+
+
+; GCN-LABEL: {{^}}mad_i64_i32_sextops_i32_i128:
+; CI: v_mad_u64_u32
+; CI: v_mad_u64_u32
+; CI: v_mad_i64_i32
+; CI: v_mad_u64_u32
+
+
+; SI-NOT: v_mad_
+define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
+  %sext0 = sext i32 %arg0 to i128
+  %sext1 = sext i32 %arg1 to i128
+  %mul = mul i128 %sext0, %sext1
+  %mad = add i128 %mul, %arg2
+  ret i128 %mad
+}
+
+; GCN-LABEL: {{^}}mad_i64_i32_sextops_i32_i63:
+; CI: v_lshl_b64
+; CI: v_ashr
+; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
+
+; SI-NOT: v_mad_u64_u32
+define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 {
+  %sext0 = sext i32 %arg0 to i63
+  %sext1 = sext i32 %arg1 to i63
+  %mul = mul i63 %sext0, %sext1
+  %mad = add i63 %mul, %arg2
+  ret i63 %mad
+}
+
+; GCN-LABEL: {{^}}mad_i64_i32_sextops_i31_i63:
+; CI: v_lshl_b64
+; CI: v_ashr_i64
+; CI: v_bfe_i32 v1, v1, 0, 31
+; CI: v_bfe_i32 v0, v0, 0, 31
+; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v1, v[2:3]
+define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
+  %sext0 = sext i31 %arg0 to i63
+  %sext1 = sext i31 %arg1 to i63
+  %mul = mul i63 %sext0, %sext1
+  %mad = add i63 %mul, %arg2
+  ret i63 %mad
+}
+
+; GCN-LABEL: {{^}}mad_u64_u32_bitops:
+; CI: v_mad_u64_u32 v[0:1], s[6:7], v0, v2, v[4:5]
+define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
+  %trunc.lhs = and i64 %arg0, 4294967295
+  %trunc.rhs = and i64 %arg1, 4294967295
+  %mul = mul i64 %trunc.lhs, %trunc.rhs
+  %add = add i64 %mul, %arg2
+  ret i64 %add
+}
+
+; GCN-LABEL: {{^}}mad_u64_u32_bitops_lhs_mask_small:
+; GCN-NOT: v_mad_
+define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
+  %trunc.lhs = and i64 %arg0, 8589934591
+  %trunc.rhs = and i64 %arg1, 4294967295
+  %mul = mul i64 %trunc.lhs, %trunc.rhs
+  %add = add i64 %mul, %arg2
+  ret i64 %add
+}
+
+; GCN-LABEL: {{^}}mad_u64_u32_bitops_rhs_mask_small:
+; GCN-NOT: v_mad_
+define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
+  %trunc.lhs = and i64 %arg0, 4294967295
+  %trunc.rhs = and i64 %arg1, 8589934591
+  %mul = mul i64 %trunc.lhs, %trunc.rhs
+  %add = add i64 %mul, %arg2
+  ret i64 %add
+}
+
+; GCN-LABEL: {{^}}mad_i64_i32_bitops:
+; CI: v_mad_i64_i32 v[0:1], s[6:7], v0, v2, v[4:5]
+; SI-NOT: v_mad_
+define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
+  %shl.lhs = shl i64 %arg0, 32
+  %trunc.lhs = ashr i64 %shl.lhs, 32
+  %shl.rhs = shl i64 %arg1, 32
+  %trunc.rhs = ashr i64 %shl.rhs, 32
+  %mul = mul i64 %trunc.lhs, %trunc.rhs
+  %add = add i64 %mul, %arg2
+  ret i64 %add
+}
+
+; Example from bug report
+; GCN-LABEL: {{^}}mad_i64_i32_unpack_i64ops:
+; CI: v_mad_u64_u32 v[0:1], s[6:7], v1, v0, v[0:1]
+; SI-NOT: v_mad_u64_u32
+define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
+  %tmp4 = lshr i64 %arg0, 32
+  %tmp5 = and i64 %arg0, 4294967295
+  %mul = mul nuw i64 %tmp4, %tmp5
+  %mad = add i64 %mul, %arg0
+  ret i64 %mad
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/madak.ll b/test/CodeGen/AMDGPU/madak.ll
index 77c35fac8b5de..cc0833393380c 100644
--- a/test/CodeGen/AMDGPU/madak.ll
+++ b/test/CodeGen/AMDGPU/madak.ll
@@ -34,7 +34,7 @@ define amdgpu_kernel void @madak_f32(float addrspace(1)* noalias %out, float add
 ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4
 ; GCN-DAG: buffer_load_dword [[VC:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8
 ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000
-; GCN-DAG: v_mad_f32 {{v[0-9]+}}, [[VA]], [[VB]], [[VK]]
+; GCN-DAG: v_madak_f32 {{v[0-9]+}}, [[VA]], [[VB]], 0x41200000
 ; GCN-DAG: v_mac_f32_e32 [[VK]], [[VA]], [[VC]]
 ; GCN: s_endpgm
 define amdgpu_kernel void @madak_2_use_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind {
diff --git a/test/CodeGen/AMDGPU/max.i16.ll b/test/CodeGen/AMDGPU/max.i16.ll
index abd75258c4d4f..8bfb616daa626 100644
--- a/test/CodeGen/AMDGPU/max.i16.ll
+++ b/test/CodeGen/AMDGPU/max.i16.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=VIPLUS %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=VIPLUS %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=VIPLUS %s
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_imax_sge_i16:
diff --git a/test/CodeGen/AMDGPU/mem-builtins.ll b/test/CodeGen/AMDGPU/mem-builtins.ll
index 1cbd0c3275106..c7607a32f1b4f 100644
--- a/test/CodeGen/AMDGPU/mem-builtins.ll
+++ b/test/CodeGen/AMDGPU/mem-builtins.ll
@@ -1,4 +1,5 @@
-; RUN: not llc -march=amdgcn -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+; RUN: not llc -march=r600 < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
 
 declare i32 @memcmp(i8 addrspace(1)* readonly nocapture, i8 addrspace(1)* readonly nocapture, i64) #0
 declare i8 addrspace(1)* @memchr(i8 addrspace(1)* readonly nocapture, i32, i64) #1
@@ -9,6 +10,9 @@ declare i32 @strcmp(i8* nocapture, i8* nocapture) #1
 
 
 ; ERROR: error: <unknown>:0:0: in function test_memcmp void (i8 addrspace(1)*, i8 addrspace(1)*, i32*): unsupported call to function memcmp
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, memcmp@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, memcmp@rel32@hi+4
 define amdgpu_kernel void @test_memcmp(i8 addrspace(1)* %x, i8 addrspace(1)* %y, i32* nocapture %p) #0 {
 entry:
   %cmp = tail call i32 @memcmp(i8 addrspace(1)* %x, i8 addrspace(1)* %y, i64 2)
@@ -17,6 +21,9 @@ entry:
 }
 
 ; ERROR: error: <unknown>:0:0: in function test_memchr void (i8 addrspace(1)*, i32, i64): unsupported call to function memchr
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, memchr@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, memchr@rel32@hi+4
 define amdgpu_kernel void @test_memchr(i8 addrspace(1)* %src, i32 %char, i64 %len) #0 {
   %res = call i8 addrspace(1)* @memchr(i8 addrspace(1)* %src, i32 %char, i64 %len)
   store volatile i8 addrspace(1)* %res, i8 addrspace(1)* addrspace(1)* undef
@@ -24,6 +31,9 @@ define amdgpu_kernel void @test_memchr(i8 addrspace(1)* %src, i32 %char, i64 %le
 }
 
 ; ERROR: error: <unknown>:0:0: in function test_strcpy void (i8*, i8*): unsupported call to function strcpy
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, strcpy@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, strcpy@rel32@hi+4
 define amdgpu_kernel void @test_strcpy(i8* %dst, i8* %src) #0 {
   %res = call i8* @strcpy(i8* %dst, i8* %src)
   store volatile i8* %res, i8* addrspace(1)* undef
@@ -31,6 +41,9 @@ define amdgpu_kernel void @test_strcpy(i8* %dst, i8* %src) #0 {
 }
 
 ; ERROR: error: <unknown>:0:0: in function test_strcmp void (i8*, i8*): unsupported call to function strcmp
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, strcmp@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, strcmp@rel32@hi+4
 define amdgpu_kernel void @test_strcmp(i8* %src0, i8* %src1) #0 {
   %res = call i32 @strcmp(i8* %src0, i8* %src1)
   store volatile i32 %res, i32 addrspace(1)* undef
@@ -38,6 +51,9 @@ define amdgpu_kernel void @test_strcmp(i8* %src0, i8* %src1) #0 {
 }
 
 ; ERROR: error: <unknown>:0:0: in function test_strlen void (i8*): unsupported call to function strlen
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, strlen@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, strlen@rel32@hi+4
 define amdgpu_kernel void @test_strlen(i8* %src) #0 {
   %res = call i32 @strlen(i8* %src)
   store volatile i32 %res, i32 addrspace(1)* undef
@@ -45,6 +61,9 @@ define amdgpu_kernel void @test_strlen(i8* %src) #0 {
 }
 
 ; ERROR: error: <unknown>:0:0: in function test_strnlen void (i8*, i32): unsupported call to function strnlen
+
+; GCN: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, strnlen@rel32@lo+4
+; GCN: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, strnlen@rel32@hi+4
 define amdgpu_kernel void @test_strnlen(i8* %src, i32 %size) #0 {
   %res = call i32 @strnlen(i8* %src, i32 %size)
   store volatile i32 %res, i32 addrspace(1)* undef
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll
new file mode 100644
index 0000000000000..15fff2ac0e1d7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll
@@ -0,0 +1,652 @@
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK-LABEL: {{^}}system_monotonic_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @system_monotonic_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in monotonic monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acquire_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acquire_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acquire monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_release_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @system_release_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in release monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acq_rel_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acq_rel_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acq_rel monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_seq_cst_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_seq_cst_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acquire_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acquire_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acquire acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_release_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_release_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in release acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acq_rel_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acq_rel_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in acq_rel acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_seq_cst_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_seq_cst_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_seq_cst_seq_cst
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_seq_cst_seq_cst(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in seq_cst seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_monotonic_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_monotonic_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acquire_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acquire_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_release_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_release_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acq_rel_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acq_rel_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_seq_cst_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_seq_cst_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acquire_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acquire_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_release_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_release_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acq_rel_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acq_rel_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_seq_cst_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_seq_cst_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_seq_cst_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_seq_cst_seq_cst(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_monotonic_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_monotonic_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acquire_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acquire_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_release_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_release_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acq_rel_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acq_rel_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_seq_cst_monotonic
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_seq_cst_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acquire_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acquire_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_release_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_release_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acq_rel_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acq_rel_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_seq_cst_acquire
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_seq_cst_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_seq_cst_seq_cst
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_seq_cst_seq_cst(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_monotonic_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_monotonic_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acquire_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acquire_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_release_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_release_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acq_rel_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acq_rel_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_seq_cst_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_seq_cst_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acquire_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acquire_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_release_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_release_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acq_rel_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acq_rel_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_seq_cst_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_seq_cst_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_seq_cst_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_seq_cst_seq_cst(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_monotonic_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_monotonic_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acquire_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acquire_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_release_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_release_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acq_rel_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acq_rel_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_seq_cst_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_seq_cst_monotonic(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acquire_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acquire_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_release_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_release_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acq_rel_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acq_rel_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_seq_cst_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_seq_cst_acquire(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_seq_cst_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_seq_cst_seq_cst(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll
new file mode 100644
index 0000000000000..c8e920a1854aa
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll
@@ -0,0 +1,225 @@
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=GCN -check-prefix=GFX8 %s
+
+; FUNC-LABEL: {{^}}system_acquire
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GFX6:       s_waitcnt vmcnt(0){{$}}
+; GFX6-NEXT:  buffer_wbinvl1{{$}}
+; GFX8:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @system_acquire() {
+entry:
+  fence acquire
+  ret void
+}
+
+; FUNC-LABEL: {{^}}system_release
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @system_release() {
+entry:
+  fence release
+  ret void
+}
+
+; FUNC-LABEL: {{^}}system_acq_rel
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX6:       buffer_wbinvl1{{$}}
+; GFX8:       buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @system_acq_rel() {
+entry:
+  fence acq_rel
+  ret void
+}
+
+; FUNC-LABEL: {{^}}system_seq_cst
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX6:       buffer_wbinvl1{{$}}
+; GFX8:       buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @system_seq_cst() {
+entry:
+  fence seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}singlethread_acquire
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @singlethread_acquire() {
+entry:
+  fence syncscope("singlethread") acquire
+  ret void
+}
+
+; FUNC-LABEL: {{^}}singlethread_release
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @singlethread_release() {
+entry:
+  fence syncscope("singlethread") release
+  ret void
+}
+
+; FUNC-LABEL: {{^}}singlethread_acq_rel
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @singlethread_acq_rel() {
+entry:
+  fence syncscope("singlethread") acq_rel
+  ret void
+}
+
+; FUNC-LABEL: {{^}}singlethread_seq_cst
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @singlethread_seq_cst() {
+entry:
+  fence syncscope("singlethread") seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}agent_acquire
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GFX6:       s_waitcnt vmcnt(0){{$}}
+; GFX6-NEXT:  buffer_wbinvl1{{$}}
+; GFX8:       s_waitcnt vmcnt(0){{$}}
+; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @agent_acquire() {
+entry:
+  fence syncscope("agent") acquire
+  ret void
+}
+
+; FUNC-LABEL: {{^}}agent_release
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @agent_release() {
+entry:
+  fence syncscope("agent") release
+  ret void
+}
+
+; FUNC-LABEL: {{^}}agent_acq_rel
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX6:       buffer_wbinvl1{{$}}
+; GFX8:       buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @agent_acq_rel() {
+entry:
+  fence syncscope("agent") acq_rel
+  ret void
+}
+
+; FUNC-LABEL: {{^}}agent_seq_cst
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_waitcnt vmcnt(0){{$}}
+; GFX6:       buffer_wbinvl1{{$}}
+; GFX8:       buffer_wbinvl1_vol{{$}}
+; GCN:        s_endpgm
+define amdgpu_kernel void @agent_seq_cst() {
+entry:
+  fence syncscope("agent") seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}workgroup_acquire
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @workgroup_acquire() {
+entry:
+  fence syncscope("workgroup") acquire
+  ret void
+}
+
+; FUNC-LABEL: {{^}}workgroup_release
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @workgroup_release() {
+entry:
+  fence syncscope("workgroup") release
+  ret void
+}
+
+; FUNC-LABEL: {{^}}workgroup_acq_rel
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @workgroup_acq_rel() {
+entry:
+  fence syncscope("workgroup") acq_rel
+  ret void
+}
+
+; FUNC-LABEL: {{^}}workgroup_seq_cst
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @workgroup_seq_cst() {
+entry:
+  fence syncscope("workgroup") seq_cst
+  ret void
+}
+
+; FUNC-LABEL: {{^}}wavefront_acquire
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @wavefront_acquire() {
+entry:
+  fence syncscope("wavefront") acquire
+  ret void
+}
+
+; FUNC-LABEL: {{^}}wavefront_release
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @wavefront_release() {
+entry:
+  fence syncscope("wavefront") release
+  ret void
+}
+
+; FUNC-LABEL: {{^}}wavefront_acq_rel
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @wavefront_acq_rel() {
+entry:
+  fence syncscope("wavefront") acq_rel
+  ret void
+}
+
+; FUNC-LABEL: {{^}}wavefront_seq_cst
+; GCN:        %bb.0
+; GCN-NOT:    ATOMIC_FENCE
+; GCN:        s_endpgm
+define amdgpu_kernel void @wavefront_seq_cst() {
+entry:
+  fence syncscope("wavefront") seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll b/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll
new file mode 100644
index 0000000000000..6f537c2014d52
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll
@@ -0,0 +1,302 @@
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK-LABEL: {{^}}system_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @system_monotonic(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acquire(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_release
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @system_release(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in release
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_acq_rel
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_acq_rel(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in acq_rel
+  ret void
+}
+
+; CHECK-LABEL: {{^}}system_seq_cst
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @system_seq_cst(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_monotonic(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acquire(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_release
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_release(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") release
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_acq_rel
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_acq_rel(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") acq_rel
+  ret void
+}
+
+; CHECK-LABEL: {{^}}singlethread_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @singlethread_seq_cst(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("singlethread") seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_monotonic(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acquire(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_release
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_release(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") release
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_acq_rel
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_acq_rel(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") acq_rel
+  ret void
+}
+
+; CHECK-LABEL: {{^}}agent_seq_cst
+; CHECK:       s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NEXT:  s_waitcnt vmcnt(0){{$}}
+; CHECK-NEXT:  buffer_wbinvl1_vol
+define amdgpu_kernel void @agent_seq_cst(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("agent") seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_monotonic(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acquire(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_release
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_release(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") release
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_acq_rel
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_acq_rel(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") acq_rel
+  ret void
+}
+
+; CHECK-LABEL: {{^}}workgroup_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @workgroup_seq_cst(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("workgroup") seq_cst
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_monotonic
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_monotonic(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") monotonic
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acquire
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acquire(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") acquire
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_release
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_release(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") release
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_acq_rel
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_acq_rel(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") acq_rel
+  ret void
+}
+
+; CHECK-LABEL: {{^}}wavefront_seq_cst
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
+; CHECK-NOT:   s_waitcnt vmcnt(0){{$}}
+; CHECK-NOT:   buffer_wbinvl1_vol
+define amdgpu_kernel void @wavefront_seq_cst(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("wavefront") seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll b/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll
new file mode 100644
index 0000000000000..b89bb0f78660f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-invalid-syncscope.ll
@@ -0,0 +1,43 @@
+; RUN: not llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s 2>&1 | FileCheck %s
+
+; CHECK: error: <unknown>:0:0: in function invalid_fence void (): Unsupported synchronization scope
+define amdgpu_kernel void @invalid_fence() {
+entry:
+  fence syncscope("invalid") seq_cst
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function invalid_load void (i32 addrspace(4)*, i32 addrspace(4)*): Unsupported synchronization scope
+define amdgpu_kernel void @invalid_load(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("invalid") seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function invalid_store void (i32, i32 addrspace(4)*): Unsupported synchronization scope
+define amdgpu_kernel void @invalid_store(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("invalid") seq_cst, align 4
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function invalid_cmpxchg void (i32 addrspace(4)*, i32, i32): Unsupported synchronization scope
+define amdgpu_kernel void @invalid_cmpxchg(
+    i32 addrspace(4)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(4)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(4)* %gep, i32 %old, i32 %in syncscope("invalid") seq_cst seq_cst
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: in function invalid_rmw void (i32 addrspace(4)*, i32): Unsupported synchronization scope
+define amdgpu_kernel void @invalid_rmw(
+    i32 addrspace(4)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(4)* %out, i32 %in syncscope("invalid") seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-load.ll b/test/CodeGen/AMDGPU/memory-legalizer-load.ll
new file mode 100644
index 0000000000000..57e705f2732a7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-load.ll
@@ -0,0 +1,377 @@
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+; GCN-LABEL: {{^}}system_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @system_unordered(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in unordered, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @system_monotonic(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in monotonic, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_acquire
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @system_acquire(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in acquire, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @system_seq_cst(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @singlethread_unordered(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") unordered, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @singlethread_monotonic(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") monotonic, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_acquire
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @singlethread_acquire(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") acquire, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @singlethread_seq_cst(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("singlethread") seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @agent_unordered(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") unordered, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @agent_monotonic(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") monotonic, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_acquire
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @agent_acquire(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") acquire, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
+; GCN-NEXT:  s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @agent_seq_cst(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("agent") seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @workgroup_unordered(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") unordered, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @workgroup_monotonic(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") monotonic, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_acquire
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @workgroup_acquire(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") acquire, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @workgroup_seq_cst(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("workgroup") seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @wavefront_unordered(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") unordered, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @wavefront_monotonic(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") monotonic, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_acquire
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @wavefront_acquire(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") acquire, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN-NOT:   buffer_wbinvl1_vol
+; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
+define amdgpu_kernel void @wavefront_seq_cst(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(4)* %in syncscope("wavefront") seq_cst, align 4
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_private_0
+; GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+define amdgpu_kernel void @nontemporal_private_0(
+    i32* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load i32, i32* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_private_1
+; GCN: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+define amdgpu_kernel void @nontemporal_private_1(
+    i32* %in, i32 addrspace(4)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32* %in, i32 %tid
+  %val = load i32, i32* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_global_0
+; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0x0{{$}}
+define amdgpu_kernel void @nontemporal_global_0(
+    i32 addrspace(1)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load i32, i32 addrspace(1)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_global_1
+; GFX8: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+; GFX9: global_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], off glc slc{{$}}
+define amdgpu_kernel void @nontemporal_global_1(
+    i32 addrspace(1)* %in, i32 addrspace(4)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
+  %val = load i32, i32 addrspace(1)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_local_0
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+define amdgpu_kernel void @nontemporal_local_0(
+    i32 addrspace(3)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load i32, i32 addrspace(3)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_local_1
+; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+define amdgpu_kernel void @nontemporal_local_1(
+    i32 addrspace(3)* %in, i32 addrspace(4)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(3)* %in, i32 %tid
+  %val = load i32, i32 addrspace(3)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_flat_0
+; GCN: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+define amdgpu_kernel void @nontemporal_flat_0(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load i32, i32 addrspace(4)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_flat_1
+; GCN: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
+define amdgpu_kernel void @nontemporal_flat_1(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(4)* %in, i32 %tid
+  %val = load i32, i32 addrspace(4)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(4)* %out
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-store.ll b/test/CodeGen/AMDGPU/memory-legalizer-store.ll
new file mode 100644
index 0000000000000..50df0bc166390
--- /dev/null
+++ b/test/CodeGen/AMDGPU/memory-legalizer-store.ll
@@ -0,0 +1,298 @@
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX8 %s
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+; GCN-LABEL: {{^}}system_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @system_unordered(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out unordered, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @system_monotonic(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_release
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @system_release(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out release, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}system_seq_cst
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @system_seq_cst(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out seq_cst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @singlethread_unordered(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") unordered, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @singlethread_monotonic(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_release
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @singlethread_release(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") release, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}singlethread_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @singlethread_seq_cst(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("singlethread") seq_cst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @agent_unordered(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") unordered, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @agent_monotonic(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_release
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @agent_release(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") release, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}agent_seq_cst
+; GCN:       s_waitcnt vmcnt(0){{$}}
+; GCN-NEXT:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @agent_seq_cst(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("agent") seq_cst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @workgroup_unordered(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") unordered, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @workgroup_monotonic(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_release
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @workgroup_release(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") release, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}workgroup_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @workgroup_seq_cst(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("workgroup") seq_cst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_unordered
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @wavefront_unordered(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") unordered, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_monotonic
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @wavefront_monotonic(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") monotonic, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_release
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @wavefront_release(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") release, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}wavefront_seq_cst
+; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
+; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
+define amdgpu_kernel void @wavefront_seq_cst(
+    i32 %in, i32 addrspace(4)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(4)* %out syncscope("wavefront") seq_cst, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_private_0
+; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+define amdgpu_kernel void @nontemporal_private_0(
+    i32 addrspace(4)* %in, i32* %out) {
+entry:
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  store i32 %val, i32* %out, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_private_1
+; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen glc slc{{$}}
+define amdgpu_kernel void @nontemporal_private_1(
+    i32 addrspace(4)* %in, i32* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
+  store i32 %val, i32* %out.gep, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_global_0
+; GFX8: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc slc{{$}}
+define amdgpu_kernel void @nontemporal_global_0(
+    i32 addrspace(4)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  store i32 %val, i32 addrspace(1)* %out, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_global_1
+; GFX8: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+; GFX9: global_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}, off glc slc{{$}}
+define amdgpu_kernel void @nontemporal_global_1(
+    i32 addrspace(4)* %in, i32 addrspace(1)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid
+  store i32 %val, i32 addrspace(1)* %out.gep, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_local_0
+; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+define amdgpu_kernel void @nontemporal_local_0(
+    i32 addrspace(4)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  store i32 %val, i32 addrspace(3)* %out, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_local_1
+; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
+define amdgpu_kernel void @nontemporal_local_1(
+    i32 addrspace(4)* %in, i32 addrspace(3)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(3)* %out, i32 %tid
+  store i32 %val, i32 addrspace(3)* %out.gep, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_flat_0
+; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+define amdgpu_kernel void @nontemporal_flat_0(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  store i32 %val, i32 addrspace(4)* %out, !nontemporal !0
+  ret void
+}
+
+; GCN-LABEL: {{^}}nontemporal_flat_1
+; GCN: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
+define amdgpu_kernel void @nontemporal_flat_1(
+    i32 addrspace(4)* %in, i32 addrspace(4)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(4)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(4)* %out, i32 %tid
+  store i32 %val, i32 addrspace(4)* %out.gep, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
diff --git a/test/CodeGen/AMDGPU/merge-load-store.mir b/test/CodeGen/AMDGPU/merge-load-store.mir
new file mode 100644
index 0000000000000..d61cefd5c7391
--- /dev/null
+++ b/test/CodeGen/AMDGPU/merge-load-store.mir
@@ -0,0 +1,70 @@
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s
+
+# Check that SILoadStoreOptimizer honors memory dependencies between moved
+# instructions.
+#
+# The following IR snippet would usually be optimized by the peephole optimizer.
+# However, an equivalent situation can occur with buffer instructions as well.
+
+# CHECK-LABEL: name: mem_dependency
+# CHECK: DS_READ2_B32 %0, 0, 1,
+# CHECK: DS_WRITE_B32 %0, killed %1, 64,
+# CHECK: DS_READ2_B32 %0, 16, 17,
+# CHECK: DS_WRITE_B32 killed %0, %5, 0
+
+--- |
+  define amdgpu_kernel void @mem_dependency(i32 addrspace(3)* %ptr.0) nounwind {
+    %ptr.4 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 1
+    %ptr.64 = getelementptr i32, i32 addrspace(3)* %ptr.0, i32 16
+    %1 = load i32, i32 addrspace(3)* %ptr.0
+    store i32 %1, i32 addrspace(3)* %ptr.64
+    %2 = load i32, i32 addrspace(3)* %ptr.64
+    %3 = load i32, i32 addrspace(3)* %ptr.4
+    %4 = add i32 %2, %3
+    store i32 %4, i32 addrspace(3)* %ptr.0
+    ret void
+  }
+...
+---
+name:            mem_dependency
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%1' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    0
+  adjustsStack:    false
+  hasCalls:        false
+  maxCallFrameSize: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+body:             |
+  bb.0:
+    liveins: %vgpr0
+
+    %1:vgpr_32 = COPY %vgpr0
+    %m0 = S_MOV_B32 -1
+    %2:vgpr_32 = DS_READ_B32 %1, 0, 0, implicit %m0, implicit %exec :: (load 4 from %ir.ptr.0)
+    DS_WRITE_B32 %1, killed %2, 64, 0, implicit %m0, implicit %exec :: (store 4 into %ir.ptr.64)
+
+    ; Make this load unmergeable, to tempt SILoadStoreOptimizer into merging the
+    ; other two loads.
+    %6:vreg_64 = DS_READ2_B32 %1, 16, 17, 0, implicit %m0, implicit %exec :: (load 8 from %ir.ptr.64, align 4)
+    %3:vgpr_32 = COPY %6.sub0
+    %4:vgpr_32 = DS_READ_B32 %1, 4, 0, implicit %m0, implicit %exec :: (load 4 from %ir.ptr.4)
+    %5:vgpr_32 = V_ADD_I32_e32 killed %3, killed %4, implicit-def %vcc, implicit %exec
+    DS_WRITE_B32 killed %1, %5, 0, 0, implicit killed %m0, implicit %exec :: (store 4 into %ir.ptr.0)
+    S_ENDPGM
+
+...
diff --git a/test/CodeGen/AMDGPU/merge-stores.ll b/test/CodeGen/AMDGPU/merge-stores.ll
index 6b0ec483247cf..2b2f67cd1b3d5 100644
--- a/test/CodeGen/AMDGPU/merge-stores.ll
+++ b/test/CodeGen/AMDGPU/merge-stores.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
 
 ; This test is mostly to test DAG store merging, so disable the vectorizer.
 ; Run with devices with different unaligned load restrictions.
@@ -10,8 +10,7 @@
 
 
 ; GCN-LABEL: {{^}}merge_global_store_2_constants_i8:
-; GCN: buffer_store_byte
-; GCN: buffer_store_byte
+; GCN: buffer_store_short
 ; GCN: s_endpgm
 define amdgpu_kernel void @merge_global_store_2_constants_i8(i8 addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr i8, i8 addrspace(1)* %out, i32 1
@@ -237,10 +236,8 @@ define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_2_adjacent_loads_shuffle_i32:
-; GCN: buffer_load_dword v
-; GCN: buffer_load_dword v
-; GCN: buffer_store_dword v
-; GCN: buffer_store_dword v
+; GCN: buffer_load_dwordx2 v
+; GCN: buffer_store_dwordx2 v
 define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
   %in.gep.1 = getelementptr i32, i32 addrspace(1)* %in, i32 1
@@ -379,15 +376,9 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(i32 a
 ; should catch this?
 
 ; GCN-LABEL: {{^}}merge_global_store_4_adjacent_loads_shuffle_i32:
-; GCN: buffer_load_dword v
-; GCN: buffer_load_dword v
-; GCN: buffer_load_dword v
-; GCN: buffer_load_dword v
+; GCN: buffer_load_dwordx4 v
 ; GCN: s_barrier
-; GCN: buffer_store_dword v
-; GCN: buffer_store_dword v
-; GCN: buffer_store_dword v
-; GCN: buffer_store_dword v
+; GCN: buffer_store_dwordx4 v
 define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
   %out.gep.2 = getelementptr i32, i32 addrspace(1)* %out, i32 2
@@ -489,8 +480,7 @@ define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(i32 addr
 }
 
 ; GCN-LABEL: {{^}}merge_local_store_2_constants_i8:
-; GCN: ds_write_b8
-; GCN: ds_write_b8
+; GCN: ds_write_b16
 ; GCN: s_endpgm
 define amdgpu_kernel void @merge_local_store_2_constants_i8(i8 addrspace(3)* %out) #0 {
   %out.gep.1 = getelementptr i8, i8 addrspace(3)* %out, i32 1
diff --git a/test/CodeGen/AMDGPU/min.ll b/test/CodeGen/AMDGPU/min.ll
index 60e43f8fb2a7d..6a1cb68d130e3 100644
--- a/test/CodeGen/AMDGPU/min.ll
+++ b/test/CodeGen/AMDGPU/min.ll
@@ -1,7 +1,7 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}v_test_imin_sle_i32:
 ; GCN: v_min_i32_e32
@@ -348,8 +348,8 @@ define amdgpu_kernel void @v_test_umin_ult_i32(i32 addrspace(1)* %out, i32 addrs
 ; SI: buffer_load_ubyte
 ; SI: v_min_u32_e32
 
-; GFX89: flat_load_ubyte
-; GFX89: flat_load_ubyte
+; GFX89: {{flat|global}}_load_ubyte
+; GFX89: {{flat|global}}_load_ubyte
 ; GFX89: v_min_u16_e32
 
 ; EG: MIN_UINT
diff --git a/test/CodeGen/AMDGPU/missing-store.ll b/test/CodeGen/AMDGPU/missing-store.ll
index 83c2a911a5ce3..9dd9c640f4b77 100644
--- a/test/CodeGen/AMDGPU/missing-store.ll
+++ b/test/CodeGen/AMDGPU/missing-store.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=SI %s
 
 @ptr_load = addrspace(3) global i32 addrspace(2)* undef, align 8
 
diff --git a/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll b/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
index e1fb00a1de307..5bc59c9f636b2 100644
--- a/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
+++ b/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll
@@ -6,11 +6,11 @@
 ; FIXME: We should be able to use the SGPR directly as src0 to v_add_i32
 
 ; GCN-LABEL: {{^}}clobber_vgpr_pair_pointer_add:
-; GCN-DAG: s_load_dwordx2 s{{\[}}[[ARG1LO:[0-9]+]]:[[ARG1HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0{{$}}
-; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[LDPTRLO:[0-9]+]]:[[LDPTRHI:[0-9]+]]{{\]}}
+; GCN: s_load_dwordx2 s{{\[}}[[ARG1LO:[0-9]+]]:[[ARG1HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0{{$}}
 
 ; GCN-NOT: v_mov_b32
 ; GCN: v_mov_b32_e32 v[[VARG1LO:[0-9]+]], s[[ARG1LO]]
+; GCN: buffer_load_dwordx2 v{{\[}}[[LDPTRLO:[0-9]+]]:[[LDPTRHI:[0-9]+]]{{\]}}
 ; GCN-NOT: v_mov_b32
 ; GCN: v_mov_b32_e32 v[[VARG1HI:[0-9]+]], s[[ARG1HI]]
 ; GCN-NOT: v_mov_b32
diff --git a/test/CodeGen/AMDGPU/mubuf-offset-private.ll b/test/CodeGen/AMDGPU/mubuf-offset-private.ll
index 742c4f8af85d9..23bd2e4bc8239 100644
--- a/test/CodeGen/AMDGPU/mubuf-offset-private.ll
+++ b/test/CodeGen/AMDGPU/mubuf-offset-private.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SICIVI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; Test addressing modes when the scratch base is not a frame index.
 
@@ -133,4 +133,23 @@ define amdgpu_kernel void @store_private_offset_i8_max_offset_plus2() #0 {
   ret void
 }
 
+; MUBUF used for stack access has bounds checking enabled before gfx9,
+; so a possibly negative base index can't be used for the vgpr offset.
+
+; GCN-LABEL: {{^}}store_private_unknown_bits_vaddr:
+; SICIVI: v_add_{{i|u}}32_e32 [[ADDR0:v[0-9]+]], vcc, 4
+; SICIVI: v_add_{{i|u}}32_e32 [[ADDR1:v[0-9]+]], vcc, 32, [[ADDR0]]
+; SICIVI: buffer_store_dword v{{[0-9]+}}, [[ADDR1]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}}
+
+; GFX9: v_add_u32_e32 [[ADDR:v[0-9]+]], 4,
+; GFX9: buffer_store_dword v{{[0-9]+}}, [[ADDR]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:32
+define amdgpu_kernel void @store_private_unknown_bits_vaddr() #0 {
+  %alloca = alloca [16 x i32], align 4
+  %vaddr = load volatile i32, i32 addrspace(1)* undef
+  %vaddr.off = add i32 %vaddr, 8
+  %gep = getelementptr inbounds [16 x i32], [16 x i32]* %alloca, i32 0, i32 %vaddr.off
+  store volatile i32 9, i32* %gep
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/mul.ll b/test/CodeGen/AMDGPU/mul.ll
index a0290789175d3..b35eefaaa0755 100644
--- a/test/CodeGen/AMDGPU/mul.ll
+++ b/test/CodeGen/AMDGPU/mul.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG %s -check-prefix=FUNC
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; mul24 and mad24 are affected
 
@@ -8,8 +8,8 @@
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
 define amdgpu_kernel void @test_mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
@@ -26,10 +26,10 @@ define amdgpu_kernel void @test_mul_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 ; EG: MULLO_INT {{\*? *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; SI: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
 define amdgpu_kernel void @v_mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
@@ -41,10 +41,10 @@ define amdgpu_kernel void @v_mul_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> a
 }
 
 ; FUNC-LABEL: {{^}}s_trunc_i64_mul_to_i32:
-; SI: s_load_dword
-; SI: s_load_dword
-; SI: s_mul_i32
-; SI: buffer_store_dword
+; GCN: s_load_dword
+; GCN: s_load_dword
+; GCN: s_mul_i32
+; GCN: buffer_store_dword
 define amdgpu_kernel void @s_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 %a, i64 %b) {
   %mul = mul i64 %b, %a
   %trunc = trunc i64 %mul to i32
@@ -53,10 +53,10 @@ define amdgpu_kernel void @s_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 %a
 }
 
 ; FUNC-LABEL: {{^}}v_trunc_i64_mul_to_i32:
-; SI: s_load_dword
-; SI: s_load_dword
-; SI: v_mul_lo_i32
-; SI: buffer_store_dword
+; GCN: s_load_dword
+; GCN: s_load_dword
+; GCN: v_mul_lo_i32
+; GCN: buffer_store_dword
 define amdgpu_kernel void @v_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %b = load i64, i64 addrspace(1)* %bptr, align 8
@@ -71,8 +71,8 @@ define amdgpu_kernel void @v_trunc_i64_mul_to_i32(i32 addrspace(1)* %out, i64 ad
 ; FUNC-LABEL: {{^}}mul64_sext_c:
 ; EG-DAG: MULLO_INT
 ; EG-DAG: MULHI_INT
-; SI-DAG: s_mul_i32
-; SI-DAG: v_mul_hi_i32
+; GCN-DAG: s_mul_i32
+; GCN-DAG: v_mul_hi_i32
 define amdgpu_kernel void @mul64_sext_c(i64 addrspace(1)* %out, i32 %in) {
 entry:
   %0 = sext i32 %in to i64
@@ -84,9 +84,9 @@ entry:
 ; FUNC-LABEL: {{^}}v_mul64_sext_c:
 ; EG-DAG: MULLO_INT
 ; EG-DAG: MULHI_INT
-; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_mul_hi_i32
-; SI: s_endpgm
+; GCN-DAG: v_mul_lo_i32
+; GCN-DAG: v_mul_hi_i32
+; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
@@ -96,9 +96,9 @@ define amdgpu_kernel void @v_mul64_sext_c(i64 addrspace(1)* %out, i32 addrspace(
 }
 
 ; FUNC-LABEL: {{^}}v_mul64_sext_inline_imm:
-; SI-DAG: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
-; SI-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
-; SI: s_endpgm
+; GCN-DAG: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; GCN-DAG: v_mul_hi_i32 v{{[0-9]+}}, v{{[0-9]+}}, 9
+; GCN: s_endpgm
 define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %val = load i32, i32 addrspace(1)* %in, align 4
   %ext = sext i32 %val to i64
@@ -108,12 +108,12 @@ define amdgpu_kernel void @v_mul64_sext_inline_imm(i64 addrspace(1)* %out, i32 a
 }
 
 ; FUNC-LABEL: {{^}}s_mul_i32:
-; SI: s_load_dword [[SRC0:s[0-9]+]],
-; SI: s_load_dword [[SRC1:s[0-9]+]],
-; SI: s_mul_i32 [[SRESULT:s[0-9]+]], [[SRC0]], [[SRC1]]
-; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
-; SI: buffer_store_dword [[VRESULT]],
-; SI: s_endpgm
+; GCN: s_load_dword [[SRC0:s[0-9]+]],
+; GCN: s_load_dword [[SRC1:s[0-9]+]],
+; GCN: s_mul_i32 [[SRESULT:s[0-9]+]], [[SRC0]], [[SRC1]]
+; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
+; GCN: buffer_store_dword [[VRESULT]],
+; GCN: s_endpgm
 define amdgpu_kernel void @s_mul_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
   %mul = mul i32 %a, %b
   store i32 %mul, i32 addrspace(1)* %out, align 4
@@ -121,7 +121,7 @@ define amdgpu_kernel void @s_mul_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nou
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i32:
-; SI: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_mul_lo_i32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 define amdgpu_kernel void @v_mul_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %a = load i32, i32 addrspace(1)* %in
@@ -146,7 +146,7 @@ define amdgpu_kernel void @s_mul_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nou
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i64:
-; SI: v_mul_lo_i32
+; GCN: v_mul_lo_i32
 define amdgpu_kernel void @v_mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) {
   %a = load i64, i64 addrspace(1)* %aptr, align 8
   %b = load i64, i64 addrspace(1)* %bptr, align 8
@@ -156,7 +156,7 @@ define amdgpu_kernel void @v_mul_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %
 }
 
 ; FUNC-LABEL: {{^}}mul32_in_branch:
-; SI: s_mul_i32
+; GCN: s_mul_i32
 define amdgpu_kernel void @mul32_in_branch(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %a, i32 %b, i32 %c) {
 entry:
   %0 = icmp eq i32 %a, 0
@@ -177,9 +177,9 @@ endif:
 }
 
 ; FUNC-LABEL: {{^}}mul64_in_branch:
-; SI-DAG: s_mul_i32
-; SI-DAG: v_mul_hi_u32
-; SI: s_endpgm
+; GCN-DAG: s_mul_i32
+; GCN-DAG: v_mul_hi_u32
+; GCN: s_endpgm
 define amdgpu_kernel void @mul64_in_branch(i64 addrspace(1)* %out, i64 addrspace(1)* %in, i64 %a, i64 %b, i64 %c) {
 entry:
   %0 = icmp eq i64 %a, 0
@@ -201,29 +201,36 @@ endif:
 
 ; FIXME: Load dwordx4
 ; FUNC-LABEL: {{^}}s_mul_i128:
-; SI: s_load_dwordx2
-; SI: s_load_dwordx2
-; SI: s_load_dwordx2
-; SI: s_load_dwordx2
+; GCN: s_load_dwordx2
+; GCN: s_load_dwordx2
+; GCN: s_load_dwordx2
+; GCN: s_load_dwordx2
 
 ; SI: v_mul_hi_u32
 ; SI: v_mul_hi_u32
 ; SI: s_mul_i32
 ; SI: v_mul_hi_u32
 ; SI: s_mul_i32
+; SI: s_mul_i32
+
 ; SI-DAG: s_mul_i32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: s_mul_i32
 ; SI-DAG: s_mul_i32
 ; SI-DAG: v_mul_hi_u32
-; SI: s_mul_i32
-; SI: s_mul_i32
-; SI: s_mul_i32
-; SI: s_mul_i32
-; SI: s_mul_i32
 
-; SI: buffer_store_dwordx4
+; VI: s_mul_i32
+; VI: v_mad_u64_u32
+; VI: s_mul_i32
+; VI: v_mul_hi_u32
+; VI: v_mad_u64_u32
+; VI: v_mul_hi_u32
+; VI: v_mad_u64_u32
+
+
+
+; GCN: buffer_store_dwordx4
 define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, i128 %a, i128 %b) nounwind #0 {
   %mul = mul i128 %a, %b
   store i128 %mul, i128 addrspace(1)* %out
@@ -231,8 +238,8 @@ define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, i128 %a, i128 %b)
 }
 
 ; FUNC-LABEL: {{^}}v_mul_i128:
-; SI: {{buffer|flat}}_load_dwordx4
-; SI: {{buffer|flat}}_load_dwordx4
+; GCN: {{buffer|flat}}_load_dwordx4
+; GCN: {{buffer|flat}}_load_dwordx4
 
 ; SI-DAG: v_mul_lo_i32
 ; SI-DAG: v_mul_hi_u32
@@ -242,7 +249,8 @@ define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, i128 %a, i128 %b)
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
 ; SI-DAG: v_mul_lo_i32
-; SI: v_add_i32_e32
+; SI-DAG: v_add_i32_e32
+
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
 ; SI-DAG: v_mul_hi_u32
@@ -252,7 +260,13 @@ define amdgpu_kernel void @s_mul_i128(i128 addrspace(1)* %out, i128 %a, i128 %b)
 ; SI-DAG: v_mul_lo_i32
 ; SI-DAG: v_mul_lo_i32
 
-; SI: {{buffer|flat}}_store_dwordx4
+; VI-DAG: v_mul_lo_i32
+; VI-DAG: v_mul_hi_u32
+; VI: v_mad_u64_u32
+; VI: v_mad_u64_u32
+; VI: v_mad_u64_u32
+
+; GCN: {{buffer|flat}}_store_dwordx4
 define amdgpu_kernel void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %gep.a = getelementptr inbounds i128, i128 addrspace(1)* %aptr, i32 %tid
diff --git a/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll b/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
index ba3ff0b08bc92..b00fff7dc183b 100644
--- a/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
+++ b/test/CodeGen/AMDGPU/multi-divergent-exit-region.ll
@@ -81,13 +81,11 @@
 ; GCN-NEXT: s_or_b64 exec, exec
 ; GCN: v_cmp_ne_u32_e32 vcc, 0
 ; GCN-NEXT: s_and_saveexec_b64
-; GCN-NEXT: s_xor_b64
 
 ; GCN: ; %exit0
 ; GCN: buffer_store_dword
 
 ; GCN: ; %UnifiedReturnBlock
-; GCN-NEXT: s_or_b64 exec, exec
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @multi_divergent_region_exit_ret_ret(i32 addrspace(1)* nocapture %arg0, i32 addrspace(1)* nocapture %arg1, i32 addrspace(1)* nocapture %arg2) #0 {
 entry:
diff --git a/test/CodeGen/AMDGPU/multilevel-break.ll b/test/CodeGen/AMDGPU/multilevel-break.ll
index 15de689b953e6..8cc02d497098b 100644
--- a/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -34,7 +34,7 @@
 ; GCN-NEXT: s_andn2_b64 exec, exec, [[OR_BREAK]]
 ; GCN-NEXT: s_cbranch_execnz [[INNER_LOOP]]
 
-; GCN: ; BB#{{[0-9]+}}: ; %Flow1{{$}}
+; GCN: ; %bb.{{[0-9]+}}: ; %Flow1{{$}}
 ; GCN-NEXT: ; in Loop: Header=[[OUTER_LOOP]] Depth=1
 
 ; Ensure copy is eliminated
diff --git a/test/CodeGen/AMDGPU/nested-calls.ll b/test/CodeGen/AMDGPU/nested-calls.ll
new file mode 100644
index 0000000000000..4cefa7e3e5116
--- /dev/null
+++ b/test/CodeGen/AMDGPU/nested-calls.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 -check-prefix=VI %s
+
+; Test calls when called by other callable functions rather than
+; kernels.
+
+declare void @external_void_func_i32(i32) #0
+
+; GCN-LABEL: {{^}}test_func_call_external_void_func_i32_imm:
+; GCN: s_waitcnt
+; GCN: s_mov_b32 s5, s32
+; Spill CSR VGPR used for SGPR spilling
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:4
+; GCN-DAG: s_add_u32 s32, s32, 0x200
+; GCN-DAG: v_writelane_b32 v32, s33, 0
+; GCN-DAG: v_writelane_b32 v32, s34, 1
+; GCN-DAG: v_writelane_b32 v32, s35, 2
+
+; GCN: s_swappc_b64
+
+; GCN: v_readlane_b32 s35, v32, 2
+; GCN: v_readlane_b32 s34, v32, 1
+; GCN: v_readlane_b32 s33, v32, 0
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4
+; GCN: s_sub_u32 s32, s32, 0x200
+; GCN: s_setpc_b64
+define void @test_func_call_external_void_func_i32_imm() #0 {
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_func_call_external_void_func_i32_imm_stack_use:
+; GCN: s_waitcnt
+; GCN: s_mov_b32 s5, s32
+; GCN: s_add_u32 s32, s32, 0x1200{{$}}
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s5 offset
+; GCN: s_swappc_b64
+; GCN: s_sub_u32 s32, s32, 0x1200{{$}}
+; GCN: s_setpc_b64
+define void @test_func_call_external_void_func_i32_imm_stack_use() #0 {
+  %alloca = alloca [16 x i32], align 4
+  %gep0 = getelementptr inbounds [16 x i32], [16 x i32]* %alloca, i32 0, i32 0
+  %gep15 = getelementptr inbounds [16 x i32], [16 x i32]* %alloca, i32 0, i32 16
+  store volatile i32 0, i32* %gep0
+  store volatile i32 0, i32* %gep15
+  call void @external_void_func_i32(i32 42)
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll b/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
index 97dc67f82607b..1c5d21c16e0b3 100644
--- a/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
+++ b/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -filetype=obj < %s | llvm-readobj -relocations -symbols | FileCheck %s -check-prefix=GCN
+; RUN: llc -march=amdgcn -mcpu=tahiti -filetype=obj < %s | llvm-readobj -relocations -symbols | FileCheck %s -check-prefix=GCN
 ; RUN: llc -march=amdgcn -mcpu=tonga -filetype=obj < %s | llvm-readobj -relocations -symbols | FileCheck %s -check-prefix=GCN
 ; RUN: llc -march=r600 -mcpu=cypress -filetype=obj < %s | llvm-readobj -relocations -symbols | FileCheck %s -check-prefix=EG
 
diff --git a/test/CodeGen/AMDGPU/nullptr.ll b/test/CodeGen/AMDGPU/nullptr.ll
index 0df16da135622..b7811064fb235 100644
--- a/test/CodeGen/AMDGPU/nullptr.ll
+++ b/test/CodeGen/AMDGPU/nullptr.ll
@@ -1,113 +1,119 @@
-;RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s
+;RUN: llc < %s -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs | FileCheck -check-prefixes=CHECK,GCN %s
+;RUN: llc < %s -march=r600 -mtriple=r600---amdgiz -verify-machineinstrs | FileCheck -check-prefixes=CHECK,R600 %s
 
-%struct.S = type { i32*, i32 addrspace(1)*, i32 addrspace(2)*, i32 addrspace(3)*, i32 addrspace(4)*, i32 addrspace(5)*}
+%struct.S = type { i32 addrspace(5)*, i32 addrspace(1)*, i32 addrspace(2)*, i32 addrspace(3)*, i32*, i32 addrspace(4)*}
 
 ; CHECK-LABEL: nullptr_priv:
 ; CHECK-NEXT: .long 0
-@nullptr_priv = global i32* addrspacecast (i32 addrspace(4)* null to i32*)
+@nullptr_priv = global i32 addrspace(5)* addrspacecast (i32* null to i32 addrspace(5)*)
 
 ; CHECK-LABEL: nullptr_glob:
-; CHECK-NEXT: .quad 0
-@nullptr_glob = global i32 addrspace(1)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(1)*)
+; GCN-NEXT: .quad 0
+; R600-NEXT: .long 0
+@nullptr_glob = global i32 addrspace(1)* addrspacecast (i32* null to i32 addrspace(1)*)
 
 ; CHECK-LABEL: nullptr_const:
-; CHECK-NEXT: .quad 0
-@nullptr_const = global i32 addrspace(2)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(2)*)
+; GCN-NEXT: .quad 0
+; R600-NEXT: .long 0
+@nullptr_const = global i32 addrspace(2)* addrspacecast (i32* null to i32 addrspace(2)*)
 
 ; CHECK-LABEL: nullptr_local:
 ; CHECK-NEXT: .long -1
-@nullptr_local = global i32 addrspace(3)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(3)*)
+@nullptr_local = global i32 addrspace(3)* addrspacecast (i32* null to i32 addrspace(3)*)
 
 ; CHECK-LABEL: nullptr_region:
 ; CHECK-NEXT: .long -1
-@nullptr_region = global i32 addrspace(5)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(5)*)
+@nullptr_region = global i32 addrspace(4)* addrspacecast (i32* null to i32 addrspace(4)*)
 
 ; CHECK-LABEL: nullptr6:
-; CHECK-NEXT: .long 0
-@nullptr6 = global i32 addrspace(6)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(6)*)
+; R600-NEXT: .long 0
+@nullptr6 = global i32 addrspace(6)* addrspacecast (i32* null to i32 addrspace(6)*)
 
 ; CHECK-LABEL: nullptr7:
-; CHECK-NEXT: .long 0
-@nullptr7 = global i32 addrspace(7)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(7)*)
+; R600-NEXT: .long 0
+@nullptr7 = global i32 addrspace(7)* addrspacecast (i32* null to i32 addrspace(7)*)
 
 ; CHECK-LABEL: nullptr8:
-; CHECK-NEXT: .long 0
-@nullptr8 = global i32 addrspace(8)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(8)*)
+; R600-NEXT: .long 0
+@nullptr8 = global i32 addrspace(8)* addrspacecast (i32* null to i32 addrspace(8)*)
 
 ; CHECK-LABEL: nullptr9:
-; CHECK-NEXT: .long 0
-@nullptr9 = global i32 addrspace(9)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(9)*)
+; R600-NEXT: .long 0
+@nullptr9 = global i32 addrspace(9)* addrspacecast (i32* null to i32 addrspace(9)*)
 
 ; CHECK-LABEL: nullptr10:
-; CHECK-NEXT: .long 0
-@nullptr10 = global i32 addrspace(10)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(10)*)
+; R600-NEXT: .long 0
+@nullptr10 = global i32 addrspace(10)* addrspacecast (i32* null to i32 addrspace(10)*)
 
 ; CHECK-LABEL: nullptr11:
-; CHECK-NEXT: .long 0
-@nullptr11 = global i32 addrspace(11)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(11)*)
+; R600-NEXT: .long 0
+@nullptr11 = global i32 addrspace(11)* addrspacecast (i32* null to i32 addrspace(11)*)
 
 ; CHECK-LABEL: nullptr12:
-; CHECK-NEXT: .long 0
-@nullptr12 = global i32 addrspace(12)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(12)*)
+; R600-NEXT: .long 0
+@nullptr12 = global i32 addrspace(12)* addrspacecast (i32* null to i32 addrspace(12)*)
 
 ; CHECK-LABEL: nullptr13:
-; CHECK-NEXT: .long 0
-@nullptr13 = global i32 addrspace(13)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(13)*)
+; R600-NEXT: .long 0
+@nullptr13 = global i32 addrspace(13)* addrspacecast (i32* null to i32 addrspace(13)*)
 
 ; CHECK-LABEL: nullptr14:
-; CHECK-NEXT: .long 0
-@nullptr14 = global i32 addrspace(14)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(14)*)
+; R600-NEXT: .long 0
+@nullptr14 = global i32 addrspace(14)* addrspacecast (i32* null to i32 addrspace(14)*)
 
 ; CHECK-LABEL: nullptr15:
-; CHECK-NEXT: .long 0
-@nullptr15 = global i32 addrspace(15)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(15)*)
+; R600-NEXT: .long 0
+@nullptr15 = global i32 addrspace(15)* addrspacecast (i32* null to i32 addrspace(15)*)
 
 ; CHECK-LABEL: nullptr16:
-; CHECK-NEXT: .long 0
-@nullptr16 = global i32 addrspace(16)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(16)*)
+; R600-NEXT: .long 0
+@nullptr16 = global i32 addrspace(16)* addrspacecast (i32* null to i32 addrspace(16)*)
 
 ; CHECK-LABEL: nullptr17:
-; CHECK-NEXT: .long 0
-@nullptr17 = global i32 addrspace(17)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(17)*)
+; R600-NEXT: .long 0
+@nullptr17 = global i32 addrspace(17)* addrspacecast (i32* null to i32 addrspace(17)*)
 
 ; CHECK-LABEL: nullptr18:
-; CHECK-NEXT: .long 0
-@nullptr18 = global i32 addrspace(18)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(18)*)
+; R600-NEXT: .long 0
+@nullptr18 = global i32 addrspace(18)* addrspacecast (i32* null to i32 addrspace(18)*)
 
 ; CHECK-LABEL: nullptr19:
-; CHECK-NEXT: .long 0
-@nullptr19 = global i32 addrspace(19)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(19)*)
+; R600-NEXT: .long 0
+@nullptr19 = global i32 addrspace(19)* addrspacecast (i32* null to i32 addrspace(19)*)
 
 ; CHECK-LABEL: nullptr20:
-; CHECK-NEXT: .long 0
-@nullptr20 = global i32 addrspace(20)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(20)*)
+; R600-NEXT: .long 0
+@nullptr20 = global i32 addrspace(20)* addrspacecast (i32* null to i32 addrspace(20)*)
 
 ; CHECK-LABEL: nullptr21:
-; CHECK-NEXT: .long 0
-@nullptr21 = global i32 addrspace(21)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(21)*)
+; R600-NEXT: .long 0
+@nullptr21 = global i32 addrspace(21)* addrspacecast (i32* null to i32 addrspace(21)*)
 
 ; CHECK-LABEL: nullptr22:
-; CHECK-NEXT: .long 0
-@nullptr22 = global i32 addrspace(22)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(22)*)
+; R600-NEXT: .long 0
+@nullptr22 = global i32 addrspace(22)* addrspacecast (i32* null to i32 addrspace(22)*)
 
 ; CHECK-LABEL: nullptr23:
-; CHECK-NEXT: .long 0
-@nullptr23 = global i32 addrspace(23)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(23)*)
+; R600-NEXT: .long 0
+@nullptr23 = global i32 addrspace(23)* addrspacecast (i32* null to i32 addrspace(23)*)
 
 ; CHECK-LABEL: structWithPointers:
 ; CHECK-NEXT: .long 0
-; CHECK-NEXT: .zero 4
-; CHECK-NEXT: .quad 0
-; CHECK-NEXT: .quad 0
+; GCN-NEXT:   .zero 4
+; GCN-NEXT:   .quad 0
+; R600-NEXT:  .long 0
+; GCN-NEXT:   .quad 0
+; R600-NEXT:  .long 0
 ; CHECK-NEXT: .long -1
-; CHECK-NEXT: .zero 4
-; CHECK-NEXT: .quad 0
+; GCN-NEXT:   .zero 4
+; GCN-NEXT:   .quad 0
+; R600-NEXT:  .long 0
 ; CHECK-NEXT: .long -1
-; CHECK-NEXT: .zero 4
+; GCN-NEXT:   .zero 4
 @structWithPointers = addrspace(1) global %struct.S {
-  i32* addrspacecast (i32 addrspace(4)* null to i32*),
-  i32 addrspace(1)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(1)*),
-  i32 addrspace(2)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(2)*),
-  i32 addrspace(3)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(3)*),
-  i32 addrspace(4)* null,
-  i32 addrspace(5)* addrspacecast (i32 addrspace(4)* null to i32 addrspace(5)*)}, align 4
+  i32 addrspace(5)* addrspacecast (i32* null to i32 addrspace(5)*),
+  i32 addrspace(1)* addrspacecast (i32* null to i32 addrspace(1)*),
+  i32 addrspace(2)* addrspacecast (i32* null to i32 addrspace(2)*),
+  i32 addrspace(3)* addrspacecast (i32* null to i32 addrspace(3)*),
+  i32* null,
+  i32 addrspace(4)* addrspacecast (i32* null to i32 addrspace(4)*)}, align 4
diff --git a/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir b/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
index 0a6c8a41130d4..9702d18d9059b 100644
--- a/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
+++ b/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
@@ -3,21 +3,21 @@
 # Check that constant is in SGPR registers
 
 # GCN-LABEL: {{^}}name: const_to_sgpr{{$}}
-# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
-# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
-# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN:        %[[HI:[0-9]+]]:sreg_32_xm0 = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]]:sreg_64 = REG_SEQUENCE killed %[[LO]], %subreg.sub0, killed %[[HI]], %subreg.sub1
 # GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
 
 
 # GCN-LABEL: {{^}}name: const_to_sgpr_multiple_use{{$}}
-# GCN:        %[[HI:[0-9]+]] = S_MOV_B32 0
-# GCN-NEXT:   %[[LO:[0-9]+]] = S_MOV_B32 1048576
-# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]] = REG_SEQUENCE killed %[[LO]], 1, killed %[[HI]], 2
+# GCN:        %[[HI:[0-9]+]]:sreg_32_xm0 = S_MOV_B32 0
+# GCN-NEXT:   %[[LO:[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048576
+# GCN-NEXT:   %[[SGPR_PAIR:[0-9]+]]:sreg_64 = REG_SEQUENCE killed %[[LO]], %subreg.sub0, killed %[[HI]], %subreg.sub1
 # GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
 # GCN-NEXT:   V_CMP_LT_U64_e64 killed %{{[0-9]+}}, %[[SGPR_PAIR]], implicit %exec
 
 # GCN-LABEL: {{^}}name: const_to_sgpr_subreg{{$}}
-# GCN:       %[[OP0:[0-9]+]] = REG_SEQUENCE killed %{{[0-9]+}}, 1, killed %{{[0-9]+}}, 2
+# GCN:       %[[OP0:[0-9]+]]:vreg_64 = REG_SEQUENCE killed %{{[0-9]+}}, %subreg.sub0, killed %{{[0-9]+}}, %subreg.sub1
 # GCN-NEXT:  V_CMP_LT_U32_e64 killed %[[OP0]].sub0, 12, implicit %exec
 
 --- |
@@ -109,7 +109,7 @@ body:             |
     %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
     %6 = COPY %7
     %9 = S_MOV_B32 0
-    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %10 = REG_SEQUENCE %2, %subreg.sub0, killed %9, %subreg.sub1
     %0 = COPY %10
     %11 = COPY %10.sub0
     %12 = COPY %10.sub1
@@ -117,10 +117,10 @@ body:             |
     %14 = COPY %8.sub1
     %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
     %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
-    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %17 = REG_SEQUENCE killed %15, %subreg.sub0, killed %16, %subreg.sub1
     %18 = S_MOV_B32 0
     %19 = S_MOV_B32 1048576
-    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %20 = REG_SEQUENCE killed %19, %subreg.sub0, killed %18, %subreg.sub1
     %22 = COPY killed %20
     %21 = V_CMP_LT_U64_e64 killed %17, %22, implicit %exec
     %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
@@ -133,7 +133,7 @@ body:             |
     %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
     %25 = S_MOV_B32 61440
     %26 = S_MOV_B32 0
-    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %27 = REG_SEQUENCE killed %26, %subreg.sub0, killed %25, %subreg.sub1
     %28 = REG_SEQUENCE %6, 17, killed %27, 18
     %29 = V_MOV_B32_e32 0, implicit %exec
     %30 = COPY %24
@@ -208,7 +208,7 @@ body:             |
     %9 = S_LOAD_DWORDX2_IMM %3, 13, 0
     %6 = COPY %7
     %10 = S_MOV_B32 0
-    %11 = REG_SEQUENCE %2, 1, killed %10, 2
+    %11 = REG_SEQUENCE %2, %subreg.sub0, killed %10, %subreg.sub1
     %0 = COPY %11
     %12 = COPY %11.sub0
     %13 = COPY %11.sub1
@@ -216,15 +216,15 @@ body:             |
     %15 = COPY %8.sub1
     %16 = S_ADD_U32 %12, killed %14, implicit-def %scc
     %17 = S_ADDC_U32 %13, killed %15, implicit-def dead %scc, implicit %scc
-    %18 = REG_SEQUENCE killed %16, 1, killed %17, 2
+    %18 = REG_SEQUENCE killed %16, %subreg.sub0, killed %17, %subreg.sub1
     %19 = COPY %9.sub0
     %20 = COPY %9.sub1
     %21 = S_ADD_U32 %12, killed %19, implicit-def %scc
     %22 = S_ADDC_U32 %13, killed %20, implicit-def dead %scc, implicit %scc
-    %23 = REG_SEQUENCE killed %21, 1, killed %22, 2
+    %23 = REG_SEQUENCE killed %21, %subreg.sub0, killed %22, %subreg.sub1
     %24 = S_MOV_B32 0
     %25 = S_MOV_B32 1048576
-    %26 = REG_SEQUENCE killed %25, 1, killed %24, 2
+    %26 = REG_SEQUENCE killed %25, %subreg.sub0, killed %24, %subreg.sub1
     %28 = COPY %26
     %27 = V_CMP_LT_U64_e64 killed %18, %28, implicit %exec
     %29 = V_CMP_LT_U64_e64 killed %23, %28, implicit %exec
@@ -239,7 +239,7 @@ body:             |
     %33 = S_LSHL_B64 %0, killed %32, implicit-def dead %scc
     %34 = S_MOV_B32 61440
     %35 = S_MOV_B32 0
-    %36 = REG_SEQUENCE killed %35, 1, killed %34, 2
+    %36 = REG_SEQUENCE killed %35, %subreg.sub0, killed %34, %subreg.sub1
     %37 = REG_SEQUENCE %6, 17, killed %36, 18
     %38 = V_MOV_B32_e32 0, implicit %exec
     %39 = COPY %33
@@ -304,7 +304,7 @@ body:             |
     %8 = S_LOAD_DWORDX2_IMM %3, 11, 0
     %6 = COPY %7
     %9 = S_MOV_B32 0
-    %10 = REG_SEQUENCE %2, 1, killed %9, 2
+    %10 = REG_SEQUENCE %2, %subreg.sub0, killed %9, %subreg.sub1
     %0 = COPY %10
     %11 = COPY %10.sub0
     %12 = COPY %10.sub1
@@ -312,10 +312,10 @@ body:             |
     %14 = COPY %8.sub1
     %15 = S_ADD_U32 killed %11, killed %13, implicit-def %scc
     %16 = S_ADDC_U32 killed %12, killed %14, implicit-def dead %scc, implicit %scc
-    %17 = REG_SEQUENCE killed %15, 1, killed %16, 2
+    %17 = REG_SEQUENCE killed %15, %subreg.sub0, killed %16, %subreg.sub1
     %18 = S_MOV_B32 12
     %19 = S_MOV_B32 1048576
-    %20 = REG_SEQUENCE killed %19, 1, killed %18, 2
+    %20 = REG_SEQUENCE killed %19, %subreg.sub0, killed %18, %subreg.sub1
     %22 = COPY killed %20.sub1
     %21 = V_CMP_LT_U32_e64 killed %17.sub0, %22, implicit %exec
     %1 = SI_IF killed %21, %bb.2.bb2, implicit-def dead %exec, implicit-def dead %scc, implicit %exec
@@ -328,7 +328,7 @@ body:             |
     %24 = S_LSHL_B64 %0, killed %23, implicit-def dead %scc
     %25 = S_MOV_B32 61440
     %26 = S_MOV_B32 0
-    %27 = REG_SEQUENCE killed %26, 1, killed %25, 2
+    %27 = REG_SEQUENCE killed %26, %subreg.sub0, killed %25, %subreg.sub1
     %28 = REG_SEQUENCE %6, 17, killed %27, 18
     %29 = V_MOV_B32_e32 0, implicit %exec
     %30 = COPY %24
diff --git a/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir b/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
index b5dc9d9dac841..24e8ed8e29cbf 100644
--- a/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
+++ b/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
@@ -184,8 +184,8 @@ body:             |
     %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -241,8 +241,8 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 4, implicit %exec
     %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -298,8 +298,8 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 4, implicit %exec
     %sgpr2_sgpr3 = S_OR_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -359,8 +359,8 @@ body:             |
     BUFFER_STORE_DWORD_OFFSET %vgpr0, undef %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`)
     %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -384,7 +384,7 @@ body:             |
 # CHECK: %sgpr0_sgpr1 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
 # CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 undef %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
 # CHECK-NEXT: %exec = COPY %sgpr0_sgpr1
-# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec
+# CHECK-NEXT: SI_MASK_BRANCH %bb.2, implicit %exec
 name:            optimize_if_and_saveexec_xor_wrong_reg
 alignment:       0
 exposesReturnsTwice: false
@@ -420,8 +420,8 @@ body:             |
     %sgpr0_sgpr1 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %sgpr0_sgpr1 = S_XOR_B64 undef %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term %sgpr0_sgpr1
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1 , %sgpr4_sgpr5_sgpr6_sgpr7
@@ -443,7 +443,7 @@ body:             |
 # CHECK-NEXT: %sgpr2_sgpr3 = S_OR_B64 killed %sgpr2_sgpr3, 1, implicit-def %scc
 # CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
 # CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3
-# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec
+# CHECK-NEXT: SI_MASK_BRANCH %bb.2, implicit %exec
 
 name:            optimize_if_and_saveexec_xor_modify_copy_to_exec
 alignment:       0
@@ -479,8 +479,8 @@ body:             |
     %sgpr2_sgpr3 = S_OR_B64 killed %sgpr2_sgpr3, 1, implicit-def %scc
     %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -540,8 +540,8 @@ body:             |
     %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1, %sgpr2_sgpr3
@@ -565,7 +565,7 @@ body:             |
 # CHECK: %sgpr0_sgpr1 = COPY %exec
 # CHECK: %sgpr2_sgpr3 = S_LSHR_B64 %sgpr0_sgpr1, killed %vcc_lo, implicit-def %scc
 # CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3
-# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec
+# CHECK-NEXT: SI_MASK_BRANCH %bb.2, implicit %exec
 
 name:            optimize_if_unknown_saveexec
 alignment:       0
@@ -599,8 +599,8 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 4, implicit %exec
     %sgpr2_sgpr3 = S_LSHR_B64 %sgpr0_sgpr1, killed %vcc_lo, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -656,8 +656,8 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 4, implicit %exec
     %sgpr2_sgpr3 = S_ANDN2_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
@@ -680,7 +680,7 @@ body:             |
 # CHECK-LABEL: name: optimize_if_andn2_saveexec_no_commute{{$}}
 # CHECK: %sgpr2_sgpr3 = S_ANDN2_B64 killed %vcc, %sgpr0_sgpr1, implicit-def %scc
 # CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3
-# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec
+# CHECK-NEXT: SI_MASK_BRANCH %bb.2, implicit %exec
 name:            optimize_if_andn2_saveexec_no_commute
 alignment:       0
 exposesReturnsTwice: false
@@ -713,8 +713,8 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 4, implicit %exec
     %sgpr2_sgpr3 = S_ANDN2_B64 killed %vcc, %sgpr0_sgpr1, implicit-def %scc
     %exec = S_MOV_B64_term killed %sgpr2_sgpr3
-    SI_MASK_BRANCH %bb.2.end, implicit %exec
-    S_BRANCH %bb.1.if
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
 
   bb.1.if:
     liveins: %sgpr0_sgpr1
diff --git a/test/CodeGen/AMDGPU/pack.v2f16.ll b/test/CodeGen/AMDGPU/pack.v2f16.ll
index 5a07f7ca6ae86..c50d3f7010a40 100644
--- a/test/CodeGen/AMDGPU/pack.v2f16.ll
+++ b/test/CodeGen/AMDGPU/pack.v2f16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx901 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
 
@@ -56,8 +56,8 @@ define amdgpu_kernel void @s_pack_v2f16_imm_hi(i32 addrspace(2)* %in0) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16:
-; GFX9: flat_load_dword [[VAL0:v[0-9]+]]
-; GFX9: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9: global_load_dword [[VAL0:v[0-9]+]]
+; GFX9: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VAL0]]
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[ELT0]]
@@ -81,13 +81,13 @@ define amdgpu_kernel void @v_pack_v2f16(i32 addrspace(1)* %in0, i32 addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_user:
-; GFX9: flat_load_dword [[VAL0:v[0-9]+]]
-; GFX9: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9: global_load_dword [[VAL0:v[0-9]+]]
+; GFX9: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9: v_and_b32_e32 [[ELT0:v[0-9]+]], 0xffff, [[VAL0]]
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[ELT0]]
 
-; GFX9: v_add_i32_e32 v{{[0-9]+}}, vcc, 9, [[PACKED]]
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 9, [[PACKED]]
 define amdgpu_kernel void @v_pack_v2f16_user(i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -108,7 +108,7 @@ define amdgpu_kernel void @v_pack_v2f16_user(i32 addrspace(1)* %in0, i32 addrspa
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_imm_lo:
-; GFX9-DAG: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x1234{{$}}
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[K]]
@@ -128,7 +128,7 @@ define amdgpu_kernel void @v_pack_v2f16_imm_lo(i32 addrspace(1)* %in1) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_inline_imm_lo:
-; GFX9-DAG: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x4400{{$}}
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[K]]
@@ -149,7 +149,7 @@ define amdgpu_kernel void @v_pack_v2f16_inline_imm_lo(i32 addrspace(1)* %in1) #0
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_imm_hi:
-; GFX9-DAG: flat_load_dword [[VAL0:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]]
 
 ; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x1234
 ; GFX9: v_and_b32_e32 [[MASKED:v[0-9]+]], 0xffff, [[VAL0]]
@@ -171,7 +171,7 @@ define amdgpu_kernel void @v_pack_v2f16_imm_hi(i32 addrspace(1)* %in0) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_inline_f16imm_hi:
-; GFX9-DAG: flat_load_dword [[VAL:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL:v[0-9]+]]
 
 ; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x3c00
 ; GFX9: v_and_b32_e32 [[MASKED:v[0-9]+]], 0xffff, [[VAL]]
@@ -193,7 +193,7 @@ define amdgpu_kernel void @v_pack_v2f16_inline_f16imm_hi(i32 addrspace(1)* %in0)
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2f16_inline_imm_hi:
-; GFX9: flat_load_dword [[VAL:v[0-9]+]]
+; GFX9: global_load_dword [[VAL:v[0-9]+]]
 
 ; GFX9: v_and_b32_e32 [[MASKED:v[0-9]+]], 0xffff, [[VAL]]
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], 64, 16, [[MASKED]]
diff --git a/test/CodeGen/AMDGPU/pack.v2i16.ll b/test/CodeGen/AMDGPU/pack.v2i16.ll
index 8515fbc6dbae5..343b94b06bd08 100644
--- a/test/CodeGen/AMDGPU/pack.v2i16.ll
+++ b/test/CodeGen/AMDGPU/pack.v2i16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx901 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -mattr=-flat-for-global,-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=CI %s
 
 
 ; GCN-LABEL: {{^}}s_pack_v2i16:
@@ -52,8 +52,8 @@ define amdgpu_kernel void @s_pack_v2i16_imm_hi(i32 addrspace(2)* %in0) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16:
-; GFX9: flat_load_dword [[VAL0:v[0-9]+]]
-; GFX9: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9: global_load_dword [[VAL0:v[0-9]+]]
+; GFX9: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9: v_and_b32_e32 [[MASKED:v[0-9]+]], 0xffff, [[VAL0]]
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[MASKED]]
@@ -75,13 +75,13 @@ define amdgpu_kernel void @v_pack_v2i16(i32 addrspace(1)* %in0, i32 addrspace(1)
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16_user:
-; GFX9: flat_load_dword [[VAL0:v[0-9]+]]
-; GFX9: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9: global_load_dword [[VAL0:v[0-9]+]]
+; GFX9: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9: v_and_b32_e32 [[MASKED:v[0-9]+]], 0xffff, [[VAL0]]
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, [[MASKED]]
 
-; GFX9: v_add_i32_e32 v{{[0-9]+}}, vcc, 9, [[PACKED]]
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 9, [[PACKED]]
 define amdgpu_kernel void @v_pack_v2i16_user(i32 addrspace(1)* %in0, i32 addrspace(1)* %in1) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -100,7 +100,7 @@ define amdgpu_kernel void @v_pack_v2i16_user(i32 addrspace(1)* %in0, i32 addrspa
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16_imm_lo:
-; GFX9-DAG: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL1:v[0-9]+]]
 ; GFX9-DENORM-DAG: s_movk_i32 [[K:s[0-9]+]], 0x7b{{$}}
 
 ; GFX9-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x7b{{$}}
@@ -121,7 +121,7 @@ define amdgpu_kernel void @v_pack_v2i16_imm_lo(i32 addrspace(1)* %in1) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16_inline_imm_lo:
-; GFX9: flat_load_dword [[VAL1:v[0-9]+]]
+; GFX9: global_load_dword [[VAL1:v[0-9]+]]
 
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[VAL1]], 16, 64
 ; GFX9: ; use [[PACKED]]
@@ -139,7 +139,7 @@ define amdgpu_kernel void @v_pack_v2i16_inline_imm_lo(i32 addrspace(1)* %in1) #0
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16_imm_hi:
-; GFX9-DAG: flat_load_dword [[VAL0:v[0-9]+]]
+; GFX9-DAG: global_load_dword [[VAL0:v[0-9]+]]
 
 ; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x7b{{$}}
 ; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[K]], 16, [[VAL0]]
@@ -159,8 +159,8 @@ define amdgpu_kernel void @v_pack_v2i16_imm_hi(i32 addrspace(1)* %in0) #0 {
 }
 
 ; GCN-LABEL: {{^}}v_pack_v2i16_inline_imm_hi:
-; GFX9: flat_load_dword [[VAL:v[0-9]+]]
-; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], 7, 16, [[VAL0]]
+; GFX9: global_load_dword [[VAL:v[0-9]+]]
+; GFX9: v_lshl_or_b32 [[PACKED:v[0-9]+]], 7, 16, [[VAL]]
 ; GFX9: ; use [[PACKED]]
 define amdgpu_kernel void @v_pack_v2i16_inline_imm_hi(i32 addrspace(1)* %in0) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/test/CodeGen/AMDGPU/packed-op-sel.ll b/test/CodeGen/AMDGPU/packed-op-sel.ll
index 4970375d40d3f..a9d7f43a67d96 100644
--- a/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -228,15 +228,13 @@ bb:
   ret void
 }
 
+; FIXME: Can we avoid waitcnt between the two halves?
 ; GCN-LABEL: {{^}}fma_vector_vector_neg_scalar_lo_scalar_hi:
 ; GCN: ds_read_b32 [[VEC0:v[0-9]+]]
 ; GCN: ds_read_b32 [[VEC1:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR0:v[0-9]+]]
-; GCN: ds_read_u16 [[SCALAR1:v[0-9]+]]
-
-; FIXME: Remove and
-; GCN: v_and_b32_e32 [[SCALAR0]], 0xffff, [[SCALAR0]]
-; GCN: v_lshl_or_b32 [[PACKED:v[0-9]+]], [[SCALAR1]], 16, [[SCALAR0]]
+; GCN: ds_read_u16 [[PACKED:v[0-9]+]]
+; GCN: s_waitcnt
+; GCN: ds_read_u16_d16_hi [[PACKED]]
 
 ; GCN: v_pk_fma_f16 v{{[0-9]+}}, [[VEC0]], [[VEC1]], [[PACKED]] neg_lo:[0,0,1] neg_hi:[0,0,1]{{$}}
 define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(<2 x half> addrspace(1)* %out, <2 x half> addrspace(3)* %lds, half addrspace(3)* %arg2) #0 {
diff --git a/test/CodeGen/AMDGPU/private-memory-atomics.ll b/test/CodeGen/AMDGPU/private-memory-atomics.ll
index 9fa3051928a05..dfec8300c96d3 100644
--- a/test/CodeGen/AMDGPU/private-memory-atomics.ll
+++ b/test/CodeGen/AMDGPU/private-memory-atomics.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tahiti < %s
 ; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s
 
 ; This works because promote allocas pass replaces these with LDS atomics.
diff --git a/test/CodeGen/AMDGPU/private-memory-broken.ll b/test/CodeGen/AMDGPU/private-memory-broken.ll
deleted file mode 100644
index 9b5f655f1b523..0000000000000
--- a/test/CodeGen/AMDGPU/private-memory-broken.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; RUN: not llc -verify-machineinstrs -march=amdgcn %s -o /dev/null 2>&1 | FileCheck %s
-; RUN: not llc -verify-machineinstrs -march=amdgcn -mcpu=tonga %s -o /dev/null 2>&1 | FileCheck %s
-
-; Make sure promote alloca pass doesn't crash
-
-; CHECK: unsupported call
-
-declare i32 @foo(i32*) nounwind
-
-define amdgpu_kernel void @call_private(i32 addrspace(1)* %out, i32 %in) nounwind {
-entry:
-  %tmp = alloca [2 x i32]
-  %tmp1 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 0
-  %tmp2 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 1
-  store i32 0, i32* %tmp1
-  store i32 1, i32* %tmp2
-  %tmp3 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 %in
-  %val = call i32 @foo(i32* %tmp3) nounwind
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
diff --git a/test/CodeGen/AMDGPU/private-memory-r600.ll b/test/CodeGen/AMDGPU/private-memory-r600.ll
index 866cd16ec3b53..65e728174291a 100644
--- a/test/CodeGen/AMDGPU/private-memory-r600.ll
+++ b/test/CodeGen/AMDGPU/private-memory-r600.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck %s -check-prefix=R600 -check-prefix=FUNC
-; RUN: opt -S -mtriple=r600-unknown-unknown -mcpu=redwood -amdgpu-promote-alloca < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood < %s | FileCheck %s -check-prefix=R600 -check-prefix=FUNC
+; RUN: opt -S -mtriple=r600-unknown-unknown-amdgiz -mcpu=redwood -amdgpu-promote-alloca < %s | FileCheck -check-prefix=OPT %s
+target datalayout = "A5"
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
@@ -18,19 +19,19 @@ declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
 define amdgpu_kernel void @mova_same_clause(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #0 {
 entry:
-  %stack = alloca [5 x i32], align 4
+  %stack = alloca [5 x i32], align 4, addrspace(5)
   %0 = load i32, i32 addrspace(1)* %in, align 4
-  %arrayidx1 = getelementptr inbounds [5 x i32], [5 x i32]* %stack, i32 0, i32 %0
-  store i32 4, i32* %arrayidx1, align 4
+  %arrayidx1 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %0
+  store i32 4, i32 addrspace(5)* %arrayidx1, align 4
   %arrayidx2 = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 1
   %1 = load i32, i32 addrspace(1)* %arrayidx2, align 4
-  %arrayidx3 = getelementptr inbounds [5 x i32], [5 x i32]* %stack, i32 0, i32 %1
-  store i32 5, i32* %arrayidx3, align 4
-  %arrayidx10 = getelementptr inbounds [5 x i32], [5 x i32]* %stack, i32 0, i32 0
-  %2 = load i32, i32* %arrayidx10, align 4
+  %arrayidx3 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 %1
+  store i32 5, i32 addrspace(5)* %arrayidx3, align 4
+  %arrayidx10 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 0
+  %2 = load i32, i32 addrspace(5)* %arrayidx10, align 4
   store i32 %2, i32 addrspace(1)* %out, align 4
-  %arrayidx12 = getelementptr inbounds [5 x i32], [5 x i32]* %stack, i32 0, i32 1
-  %3 = load i32, i32* %arrayidx12
+  %arrayidx12 = getelementptr inbounds [5 x i32], [5 x i32] addrspace(5)* %stack, i32 0, i32 1
+  %3 = load i32, i32 addrspace(5)* %arrayidx12
   %arrayidx13 = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 1
   store i32 %3, i32 addrspace(1)* %arrayidx13
   ret void
@@ -49,20 +50,20 @@ entry:
 
 define amdgpu_kernel void @multiple_structs(i32 addrspace(1)* %out) #0 {
 entry:
-  %a = alloca %struct.point
-  %b = alloca %struct.point
-  %a.x.ptr = getelementptr inbounds %struct.point, %struct.point* %a, i32 0, i32 0
-  %a.y.ptr = getelementptr inbounds %struct.point, %struct.point* %a, i32 0, i32 1
-  %b.x.ptr = getelementptr inbounds %struct.point, %struct.point* %b, i32 0, i32 0
-  %b.y.ptr = getelementptr inbounds %struct.point, %struct.point* %b, i32 0, i32 1
-  store i32 0, i32* %a.x.ptr
-  store i32 1, i32* %a.y.ptr
-  store i32 2, i32* %b.x.ptr
-  store i32 3, i32* %b.y.ptr
-  %a.indirect.ptr = getelementptr inbounds %struct.point, %struct.point* %a, i32 0, i32 0
-  %b.indirect.ptr = getelementptr inbounds %struct.point, %struct.point* %b, i32 0, i32 0
-  %a.indirect = load i32, i32* %a.indirect.ptr
-  %b.indirect = load i32, i32* %b.indirect.ptr
+  %a = alloca %struct.point, addrspace(5)
+  %b = alloca %struct.point, addrspace(5)
+  %a.x.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %a, i32 0, i32 0
+  %a.y.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %a, i32 0, i32 1
+  %b.x.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %b, i32 0, i32 0
+  %b.y.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %b, i32 0, i32 1
+  store i32 0, i32 addrspace(5)* %a.x.ptr
+  store i32 1, i32 addrspace(5)* %a.y.ptr
+  store i32 2, i32 addrspace(5)* %b.x.ptr
+  store i32 3, i32 addrspace(5)* %b.y.ptr
+  %a.indirect.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %a, i32 0, i32 0
+  %b.indirect.ptr = getelementptr inbounds %struct.point, %struct.point addrspace(5)* %b, i32 0, i32 0
+  %a.indirect = load i32, i32 addrspace(5)* %a.indirect.ptr
+  %b.indirect = load i32, i32 addrspace(5)* %b.indirect.ptr
   %0 = add i32 %a.indirect, %b.indirect
   store i32 %0, i32 addrspace(1)* %out
   ret void
@@ -77,32 +78,32 @@ entry:
 
 define amdgpu_kernel void @direct_loop(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
 entry:
-  %prv_array_const = alloca [2 x i32]
-  %prv_array = alloca [2 x i32]
+  %prv_array_const = alloca [2 x i32], addrspace(5)
+  %prv_array = alloca [2 x i32], addrspace(5)
   %a = load i32, i32 addrspace(1)* %in
   %b_src_ptr = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 1
   %b = load i32, i32 addrspace(1)* %b_src_ptr
-  %a_dst_ptr = getelementptr inbounds [2 x i32], [2 x i32]* %prv_array_const, i32 0, i32 0
-  store i32 %a, i32* %a_dst_ptr
-  %b_dst_ptr = getelementptr inbounds [2 x i32], [2 x i32]* %prv_array_const, i32 0, i32 1
-  store i32 %b, i32* %b_dst_ptr
+  %a_dst_ptr = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %prv_array_const, i32 0, i32 0
+  store i32 %a, i32 addrspace(5)* %a_dst_ptr
+  %b_dst_ptr = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %prv_array_const, i32 0, i32 1
+  store i32 %b, i32 addrspace(5)* %b_dst_ptr
   br label %for.body
 
 for.body:
   %inc = phi i32 [0, %entry], [%count, %for.body]
-  %x_ptr = getelementptr inbounds [2 x i32], [2 x i32]* %prv_array_const, i32 0, i32 0
-  %x = load i32, i32* %x_ptr
-  %y_ptr = getelementptr inbounds [2 x i32], [2 x i32]* %prv_array, i32 0, i32 0
-  %y = load i32, i32* %y_ptr
+  %x_ptr = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %prv_array_const, i32 0, i32 0
+  %x = load i32, i32 addrspace(5)* %x_ptr
+  %y_ptr = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %prv_array, i32 0, i32 0
+  %y = load i32, i32 addrspace(5)* %y_ptr
   %xy = add i32 %x, %y
-  store i32 %xy, i32* %y_ptr
+  store i32 %xy, i32 addrspace(5)* %y_ptr
   %count = add i32 %inc, 1
   %done = icmp eq i32 %count, 4095
   br i1 %done, label %for.end, label %for.body
 
 for.end:
-  %value_ptr = getelementptr inbounds [2 x i32], [2 x i32]* %prv_array, i32 0, i32 0
-  %value = load i32, i32* %value_ptr
+  %value_ptr = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %prv_array, i32 0, i32 0
+  %value = load i32, i32 addrspace(5)* %value_ptr
   store i32 %value, i32 addrspace(1)* %out
   ret void
 }
@@ -112,13 +113,13 @@ for.end:
 ; R600: MOVA_INT
 define amdgpu_kernel void @short_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %0 = alloca [2 x i16]
-  %1 = getelementptr inbounds [2 x i16], [2 x i16]* %0, i32 0, i32 0
-  %2 = getelementptr inbounds [2 x i16], [2 x i16]* %0, i32 0, i32 1
-  store i16 0, i16* %1
-  store i16 1, i16* %2
-  %3 = getelementptr inbounds [2 x i16], [2 x i16]* %0, i32 0, i32 %index
-  %4 = load i16, i16* %3
+  %0 = alloca [2 x i16], addrspace(5)
+  %1 = getelementptr inbounds [2 x i16], [2 x i16] addrspace(5)* %0, i32 0, i32 0
+  %2 = getelementptr inbounds [2 x i16], [2 x i16] addrspace(5)* %0, i32 0, i32 1
+  store i16 0, i16 addrspace(5)* %1
+  store i16 1, i16 addrspace(5)* %2
+  %3 = getelementptr inbounds [2 x i16], [2 x i16] addrspace(5)* %0, i32 0, i32 %index
+  %4 = load i16, i16 addrspace(5)* %3
   %5 = sext i16 %4 to i32
   store i32 %5, i32 addrspace(1)* %out
   ret void
@@ -129,13 +130,13 @@ entry:
 ; R600: MOVA_INT
 define amdgpu_kernel void @char_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %0 = alloca [2 x i8]
-  %1 = getelementptr inbounds [2 x i8], [2 x i8]* %0, i32 0, i32 0
-  %2 = getelementptr inbounds [2 x i8], [2 x i8]* %0, i32 0, i32 1
-  store i8 0, i8* %1
-  store i8 1, i8* %2
-  %3 = getelementptr inbounds [2 x i8], [2 x i8]* %0, i32 0, i32 %index
-  %4 = load i8, i8* %3
+  %0 = alloca [2 x i8], addrspace(5)
+  %1 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %0, i32 0, i32 0
+  %2 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %0, i32 0, i32 1
+  store i8 0, i8 addrspace(5)* %1
+  store i8 1, i8 addrspace(5)* %2
+  %3 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %0, i32 0, i32 %index
+  %4 = load i8, i8 addrspace(5)* %3
   %5 = sext i8 %4 to i32
   store i32 %5, i32 addrspace(1)* %out
   ret void
@@ -150,13 +151,13 @@ entry:
 ; R600-NOT: MOV * TO.X
 define amdgpu_kernel void @work_item_info(i32 addrspace(1)* %out, i32 %in) #0 {
 entry:
-  %0 = alloca [2 x i32]
-  %1 = getelementptr inbounds [2 x i32], [2 x i32]* %0, i32 0, i32 0
-  %2 = getelementptr inbounds [2 x i32], [2 x i32]* %0, i32 0, i32 1
-  store i32 0, i32* %1
-  store i32 1, i32* %2
-  %3 = getelementptr inbounds [2 x i32], [2 x i32]* %0, i32 0, i32 %in
-  %4 = load i32, i32* %3
+  %0 = alloca [2 x i32], addrspace(5)
+  %1 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %0, i32 0, i32 0
+  %2 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %0, i32 0, i32 1
+  store i32 0, i32 addrspace(5)* %1
+  store i32 1, i32 addrspace(5)* %2
+  %3 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %0, i32 0, i32 %in
+  %4 = load i32, i32 addrspace(5)* %3
   %5 = call i32 @llvm.r600.read.tidig.x()
   %6 = add i32 %4, %5
   store i32 %6, i32 addrspace(1)* %out
@@ -171,22 +172,22 @@ entry:
 ; R600-NOT: [[CHAN]]+
 define amdgpu_kernel void @no_overlap(i32 addrspace(1)* %out, i32 %in) #0 {
 entry:
-  %0 = alloca [3 x i8], align 1
-  %1 = alloca [2 x i8], align 1
-  %2 = getelementptr inbounds [3 x i8], [3 x i8]* %0, i32 0, i32 0
-  %3 = getelementptr inbounds [3 x i8], [3 x i8]* %0, i32 0, i32 1
-  %4 = getelementptr inbounds [3 x i8], [3 x i8]* %0, i32 0, i32 2
-  %5 = getelementptr inbounds [2 x i8], [2 x i8]* %1, i32 0, i32 0
-  %6 = getelementptr inbounds [2 x i8], [2 x i8]* %1, i32 0, i32 1
-  store i8 0, i8* %2
-  store i8 1, i8* %3
-  store i8 2, i8* %4
-  store i8 1, i8* %5
-  store i8 0, i8* %6
-  %7 = getelementptr inbounds [3 x i8], [3 x i8]* %0, i32 0, i32 %in
-  %8 = getelementptr inbounds [2 x i8], [2 x i8]* %1, i32 0, i32 %in
-  %9 = load i8, i8* %7
-  %10 = load i8, i8* %8
+  %0 = alloca [3 x i8], align 1, addrspace(5)
+  %1 = alloca [2 x i8], align 1, addrspace(5)
+  %2 = getelementptr inbounds [3 x i8], [3 x i8] addrspace(5)* %0, i32 0, i32 0
+  %3 = getelementptr inbounds [3 x i8], [3 x i8] addrspace(5)* %0, i32 0, i32 1
+  %4 = getelementptr inbounds [3 x i8], [3 x i8] addrspace(5)* %0, i32 0, i32 2
+  %5 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %1, i32 0, i32 0
+  %6 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %1, i32 0, i32 1
+  store i8 0, i8 addrspace(5)* %2
+  store i8 1, i8 addrspace(5)* %3
+  store i8 2, i8 addrspace(5)* %4
+  store i8 1, i8 addrspace(5)* %5
+  store i8 0, i8 addrspace(5)* %6
+  %7 = getelementptr inbounds [3 x i8], [3 x i8] addrspace(5)* %0, i32 0, i32 %in
+  %8 = getelementptr inbounds [2 x i8], [2 x i8] addrspace(5)* %1, i32 0, i32 %in
+  %9 = load i8, i8 addrspace(5)* %7
+  %10 = load i8, i8 addrspace(5)* %8
   %11 = add i8 %9, %10
   %12 = sext i8 %11 to i32
   store i32 %12, i32 addrspace(1)* %out
@@ -195,13 +196,13 @@ entry:
 
 define amdgpu_kernel void @char_array_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %alloca = alloca [2 x [2 x i8]]
-  %gep0 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]]* %alloca, i32 0, i32 0, i32 0
-  %gep1 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]]* %alloca, i32 0, i32 0, i32 1
-  store i8 0, i8* %gep0
-  store i8 1, i8* %gep1
-  %gep2 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]]* %alloca, i32 0, i32 0, i32 %index
-  %load = load i8, i8* %gep2
+  %alloca = alloca [2 x [2 x i8]], addrspace(5)
+  %gep0 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]] addrspace(5)* %alloca, i32 0, i32 0, i32 0
+  %gep1 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]] addrspace(5)* %alloca, i32 0, i32 0, i32 1
+  store i8 0, i8 addrspace(5)* %gep0
+  store i8 1, i8 addrspace(5)* %gep1
+  %gep2 = getelementptr inbounds [2 x [2 x i8]], [2 x [2 x i8]] addrspace(5)* %alloca, i32 0, i32 0, i32 %index
+  %load = load i8, i8 addrspace(5)* %gep2
   %sext = sext i8 %load to i32
   store i32 %sext, i32 addrspace(1)* %out
   ret void
@@ -209,26 +210,26 @@ entry:
 
 define amdgpu_kernel void @i32_array_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %alloca = alloca [2 x [2 x i32]]
-  %gep0 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %alloca, i32 0, i32 0, i32 0
-  %gep1 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %alloca, i32 0, i32 0, i32 1
-  store i32 0, i32* %gep0
-  store i32 1, i32* %gep1
-  %gep2 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]]* %alloca, i32 0, i32 0, i32 %index
-  %load = load i32, i32* %gep2
+  %alloca = alloca [2 x [2 x i32]], addrspace(5)
+  %gep0 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]] addrspace(5)* %alloca, i32 0, i32 0, i32 0
+  %gep1 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]] addrspace(5)* %alloca, i32 0, i32 0, i32 1
+  store i32 0, i32 addrspace(5)* %gep0
+  store i32 1, i32 addrspace(5)* %gep1
+  %gep2 = getelementptr inbounds [2 x [2 x i32]], [2 x [2 x i32]] addrspace(5)* %alloca, i32 0, i32 0, i32 %index
+  %load = load i32, i32 addrspace(5)* %gep2
   store i32 %load, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @i64_array_array(i64 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %alloca = alloca [2 x [2 x i64]]
-  %gep0 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]]* %alloca, i32 0, i32 0, i32 0
-  %gep1 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]]* %alloca, i32 0, i32 0, i32 1
-  store i64 0, i64* %gep0
-  store i64 1, i64* %gep1
-  %gep2 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]]* %alloca, i32 0, i32 0, i32 %index
-  %load = load i64, i64* %gep2
+  %alloca = alloca [2 x [2 x i64]], addrspace(5)
+  %gep0 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]] addrspace(5)* %alloca, i32 0, i32 0, i32 0
+  %gep1 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]] addrspace(5)* %alloca, i32 0, i32 0, i32 1
+  store i64 0, i64 addrspace(5)* %gep0
+  store i64 1, i64 addrspace(5)* %gep1
+  %gep2 = getelementptr inbounds [2 x [2 x i64]], [2 x [2 x i64]] addrspace(5)* %alloca, i32 0, i32 0, i32 %index
+  %load = load i64, i64 addrspace(5)* %gep2
   store i64 %load, i64 addrspace(1)* %out
   ret void
 }
@@ -237,40 +238,40 @@ entry:
 
 define amdgpu_kernel void @struct_array_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %alloca = alloca [2 x [2 x %struct.pair32]]
-  %gep0 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]]* %alloca, i32 0, i32 0, i32 0, i32 1
-  %gep1 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]]* %alloca, i32 0, i32 0, i32 1, i32 1
-  store i32 0, i32* %gep0
-  store i32 1, i32* %gep1
-  %gep2 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]]* %alloca, i32 0, i32 0, i32 %index, i32 0
-  %load = load i32, i32* %gep2
+  %alloca = alloca [2 x [2 x %struct.pair32]], addrspace(5)
+  %gep0 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]] addrspace(5)* %alloca, i32 0, i32 0, i32 0, i32 1
+  %gep1 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]] addrspace(5)* %alloca, i32 0, i32 0, i32 1, i32 1
+  store i32 0, i32 addrspace(5)* %gep0
+  store i32 1, i32 addrspace(5)* %gep1
+  %gep2 = getelementptr inbounds [2 x [2 x %struct.pair32]], [2 x [2 x %struct.pair32]] addrspace(5)* %alloca, i32 0, i32 0, i32 %index, i32 0
+  %load = load i32, i32 addrspace(5)* %gep2
   store i32 %load, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @struct_pair32_array(i32 addrspace(1)* %out, i32 %index) #0 {
 entry:
-  %alloca = alloca [2 x %struct.pair32]
-  %gep0 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32]* %alloca, i32 0, i32 0, i32 1
-  %gep1 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32]* %alloca, i32 0, i32 1, i32 0
-  store i32 0, i32* %gep0
-  store i32 1, i32* %gep1
-  %gep2 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32]* %alloca, i32 0, i32 %index, i32 0
-  %load = load i32, i32* %gep2
+  %alloca = alloca [2 x %struct.pair32], addrspace(5)
+  %gep0 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32] addrspace(5)* %alloca, i32 0, i32 0, i32 1
+  %gep1 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32] addrspace(5)* %alloca, i32 0, i32 1, i32 0
+  store i32 0, i32 addrspace(5)* %gep0
+  store i32 1, i32 addrspace(5)* %gep1
+  %gep2 = getelementptr inbounds [2 x %struct.pair32], [2 x %struct.pair32] addrspace(5)* %alloca, i32 0, i32 %index, i32 0
+  %load = load i32, i32 addrspace(5)* %gep2
   store i32 %load, i32 addrspace(1)* %out
   ret void
 }
 
 define amdgpu_kernel void @select_private(i32 addrspace(1)* %out, i32 %in) nounwind {
 entry:
-  %tmp = alloca [2 x i32]
-  %tmp1 = getelementptr inbounds  [2 x i32], [2 x i32]* %tmp, i32 0, i32 0
-  %tmp2 = getelementptr inbounds [2 x i32], [2 x i32]* %tmp, i32 0, i32 1
-  store i32 0, i32* %tmp1
-  store i32 1, i32* %tmp2
+  %tmp = alloca [2 x i32], addrspace(5)
+  %tmp1 = getelementptr inbounds  [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %tmp2 = getelementptr inbounds [2 x i32], [2 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  store i32 0, i32 addrspace(5)* %tmp1
+  store i32 1, i32 addrspace(5)* %tmp2
   %cmp = icmp eq i32 %in, 0
-  %sel = select i1 %cmp, i32* %tmp1, i32* %tmp2
-  %load = load i32, i32* %sel
+  %sel = select i1 %cmp, i32 addrspace(5)* %tmp1, i32 addrspace(5)* %tmp2
+  %load = load i32, i32 addrspace(5)* %sel
   store i32 %load, i32 addrspace(1)* %out
   ret void
 }
@@ -283,14 +284,14 @@ entry:
 ; SI: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen
 ; SI: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen ;
 define amdgpu_kernel void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 {
-  %alloca = alloca [16 x i32]
-  %tmp0 = getelementptr inbounds [16 x i32], [16 x i32]* %alloca, i32 0, i32 %a
-  store i32 5, i32* %tmp0
-  %tmp1 = ptrtoint [16 x i32]* %alloca to i32
+  %alloca = alloca [16 x i32], addrspace(5)
+  %tmp0 = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 %a
+  store i32 5, i32 addrspace(5)* %tmp0
+  %tmp1 = ptrtoint [16 x i32] addrspace(5)* %alloca to i32
   %tmp2 = add i32 %tmp1, 5
-  %tmp3 = inttoptr i32 %tmp2 to i32*
-  %tmp4 = getelementptr inbounds i32, i32* %tmp3, i32 %b
-  %tmp5 = load i32, i32* %tmp4
+  %tmp3 = inttoptr i32 %tmp2 to i32 addrspace(5)*
+  %tmp4 = getelementptr inbounds i32, i32 addrspace(5)* %tmp3, i32 %b
+  %tmp5 = load i32, i32 addrspace(5)* %tmp4
   store i32 %tmp5, i32 addrspace(1)* %out
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll b/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
index 5b2da788a4052..deca2997100d7 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-array-aggregate.ll
@@ -1,131 +1,140 @@
-; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-promote-alloca < %s | FileCheck --check-prefix=OPT %s
-
-; Make sure that array alloca loaded and stored as multi-element aggregates are handled correctly
-; Strictly the promote-alloca pass shouldn't have to deal with this case as it is non-canonical, but
-; the pass should handle it gracefully if it is
-; The checks look for lines that previously caused issues in PromoteAlloca (non-canonical). Opt
-; should now leave these unchanged
-
-; OPT-LABEL: @promote_1d_aggr(
-; OPT: store [1 x float] %tmp3, [1 x float]* %f1
-
-%Block = type { [1 x float], i32 }
-%gl_PerVertex = type { <4 x float>, float, [1 x float], [1 x float] }
-
-@block = external addrspace(1) global %Block
-@pv = external addrspace(1) global %gl_PerVertex
-
-define amdgpu_vs void @promote_1d_aggr() #0 {
-  %i = alloca i32
-  %f1 = alloca [1 x float]
-  %tmp = getelementptr %Block, %Block addrspace(1)* @block, i32 0, i32 1
-  %tmp1 = load i32, i32 addrspace(1)* %tmp
-  store i32 %tmp1, i32* %i
-  %tmp2 = getelementptr %Block, %Block addrspace(1)* @block, i32 0, i32 0
-  %tmp3 = load [1 x float], [1 x float] addrspace(1)* %tmp2
-  store [1 x float] %tmp3, [1 x float]* %f1
-  %tmp4 = load i32, i32* %i
-  %tmp5 = getelementptr [1 x float], [1 x float]* %f1, i32 0, i32 %tmp4
-  %tmp6 = load float, float* %tmp5
-  %tmp7 = alloca <4 x float>
-  %tmp8 = load <4 x float>, <4 x float>* %tmp7
-  %tmp9 = insertelement <4 x float> %tmp8, float %tmp6, i32 0
-  %tmp10 = insertelement <4 x float> %tmp9, float %tmp6, i32 1
-  %tmp11 = insertelement <4 x float> %tmp10, float %tmp6, i32 2
-  %tmp12 = insertelement <4 x float> %tmp11, float %tmp6, i32 3
-  %tmp13 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
-  store <4 x float> %tmp12, <4 x float> addrspace(1)* %tmp13
-  ret void
-}
-
-
-; OPT-LABEL: @promote_store_aggr(
-; OPT: %tmp6 = load [2 x float], [2 x float]* %f1
-
-%Block2 = type { i32, [2 x float] }
-@block2 = external addrspace(1) global %Block2
-
-define amdgpu_vs void @promote_store_aggr() #0 {
-  %i = alloca i32
-  %f1 = alloca [2 x float]
-  %tmp = getelementptr %Block2, %Block2 addrspace(1)* @block2, i32 0, i32 0
-  %tmp1 = load i32, i32 addrspace(1)* %tmp
-  store i32 %tmp1, i32* %i
-  %tmp2 = load i32, i32* %i
-  %tmp3 = sitofp i32 %tmp2 to float
-  %tmp4 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 0
-  store float %tmp3, float* %tmp4
-  %tmp5 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 1
-  store float 2.000000e+00, float* %tmp5
-  %tmp6 = load [2 x float], [2 x float]* %f1
-  %tmp7 = getelementptr %Block2, %Block2 addrspace(1)* @block2, i32 0, i32 1
-  store [2 x float] %tmp6, [2 x float] addrspace(1)* %tmp7
-  %tmp8 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
-  store <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <4 x float> addrspace(1)* %tmp8
-  ret void
-}
-
-; OPT-LABEL: @promote_load_from_store_aggr(
-; OPT: store [2 x float] %tmp3, [2 x float]* %f1
-
-%Block3 = type { [2 x float], i32 }
-@block3 = external addrspace(1) global %Block3
-
-define amdgpu_vs void @promote_load_from_store_aggr() #0 {
-  %i = alloca i32
-  %f1 = alloca [2 x float]
-  %tmp = getelementptr %Block3, %Block3 addrspace(1)* @block3, i32 0, i32 1
-  %tmp1 = load i32, i32 addrspace(1)* %tmp
-  store i32 %tmp1, i32* %i
-  %tmp2 = getelementptr %Block3, %Block3 addrspace(1)* @block3, i32 0, i32 0
-  %tmp3 = load [2 x float], [2 x float] addrspace(1)* %tmp2
-  store [2 x float] %tmp3, [2 x float]* %f1
-  %tmp4 = load i32, i32* %i
-  %tmp5 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 %tmp4
-  %tmp6 = load float, float* %tmp5
-  %tmp7 = alloca <4 x float>
-  %tmp8 = load <4 x float>, <4 x float>* %tmp7
-  %tmp9 = insertelement <4 x float> %tmp8, float %tmp6, i32 0
-  %tmp10 = insertelement <4 x float> %tmp9, float %tmp6, i32 1
-  %tmp11 = insertelement <4 x float> %tmp10, float %tmp6, i32 2
-  %tmp12 = insertelement <4 x float> %tmp11, float %tmp6, i32 3
-  %tmp13 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
-  store <4 x float> %tmp12, <4 x float> addrspace(1)* %tmp13
-  ret void
-}
-
-; OPT-LABEL: @promote_double_aggr(
-; OPT: store [2 x double] %tmp5, [2 x double]* %s
-
-@tmp_g = external addrspace(1) global { [4 x double], <2 x double>, <3 x double>, <4 x double> }
-@frag_color = external addrspace(1) global <4 x float>
-
-define amdgpu_ps void @promote_double_aggr() #0 {
-  %s = alloca [2 x double]
-  %tmp = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, { [4 x double], <2 x double>, <3 x double>, <4 x double> } addrspace(1)* @tmp_g, i32 0, i32 0, i32 0
-  %tmp1 = load double, double addrspace(1)* %tmp
-  %tmp2 = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, { [4 x double], <2 x double>, <3 x double>, <4 x double> } addrspace(1)* @tmp_g, i32 0, i32 0, i32 1
-  %tmp3 = load double, double addrspace(1)* %tmp2
-  %tmp4 = insertvalue [2 x double] undef, double %tmp1, 0
-  %tmp5 = insertvalue [2 x double] %tmp4, double %tmp3, 1
-  store [2 x double] %tmp5, [2 x double]* %s
-  %tmp6 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
-  %tmp7 = load double, double* %tmp6
-  %tmp8 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
-  %tmp9 = load double, double* %tmp8
-  %tmp10 = fadd double %tmp7, %tmp9
-  %tmp11 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 0
-  store double %tmp10, double* %tmp11
-  %tmp12 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 0
-  %tmp13 = load double, double* %tmp12
-  %tmp14 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
-  %tmp15 = load double, double* %tmp14
-  %tmp16 = fadd double %tmp13, %tmp15
-  %tmp17 = fptrunc double %tmp16 to float
-  %tmp18 = insertelement <4 x float> undef, float %tmp17, i32 0
-  %tmp19 = insertelement <4 x float> %tmp18, float %tmp17, i32 1
-  %tmp20 = insertelement <4 x float> %tmp19, float %tmp17, i32 2
-  %tmp21 = insertelement <4 x float> %tmp20, float %tmp17, i32 3
-  store <4 x float> %tmp21, <4 x float> addrspace(1)* @frag_color
-  ret void
-}
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-promote-alloca < %s | FileCheck --check-prefix=OPT %s
+
+; Make sure that array alloca loaded and stored as multi-element aggregates are handled correctly
+; Strictly the promote-alloca pass shouldn't have to deal with this case as it is non-canonical, but
+; the pass should handle it gracefully if it is
+; The checks look for lines that previously caused issues in PromoteAlloca (non-canonical). Opt
+; should now leave these unchanged
+
+; OPT-LABEL: @promote_1d_aggr(
+; OPT: store [1 x float] %tmp3, [1 x float]* %f1
+
+%Block = type { [1 x float], i32 }
+%gl_PerVertex = type { <4 x float>, float, [1 x float], [1 x float] }
+%struct = type { i32, i32 }
+
+@block = external addrspace(1) global %Block
+@pv = external addrspace(1) global %gl_PerVertex
+
+define amdgpu_vs void @promote_1d_aggr() #0 {
+  %i = alloca i32
+  %f1 = alloca [1 x float]
+  %tmp = getelementptr %Block, %Block addrspace(1)* @block, i32 0, i32 1
+  %tmp1 = load i32, i32 addrspace(1)* %tmp
+  store i32 %tmp1, i32* %i
+  %tmp2 = getelementptr %Block, %Block addrspace(1)* @block, i32 0, i32 0
+  %tmp3 = load [1 x float], [1 x float] addrspace(1)* %tmp2
+  store [1 x float] %tmp3, [1 x float]* %f1
+  %tmp4 = load i32, i32* %i
+  %tmp5 = getelementptr [1 x float], [1 x float]* %f1, i32 0, i32 %tmp4
+  %tmp6 = load float, float* %tmp5
+  %tmp7 = alloca <4 x float>
+  %tmp8 = load <4 x float>, <4 x float>* %tmp7
+  %tmp9 = insertelement <4 x float> %tmp8, float %tmp6, i32 0
+  %tmp10 = insertelement <4 x float> %tmp9, float %tmp6, i32 1
+  %tmp11 = insertelement <4 x float> %tmp10, float %tmp6, i32 2
+  %tmp12 = insertelement <4 x float> %tmp11, float %tmp6, i32 3
+  %tmp13 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
+  store <4 x float> %tmp12, <4 x float> addrspace(1)* %tmp13
+  ret void
+}
+
+
+; OPT-LABEL: @promote_store_aggr(
+; OPT: %tmp6 = load [2 x float], [2 x float]* %f1
+
+%Block2 = type { i32, [2 x float] }
+@block2 = external addrspace(1) global %Block2
+
+define amdgpu_vs void @promote_store_aggr() #0 {
+  %i = alloca i32
+  %f1 = alloca [2 x float]
+  %tmp = getelementptr %Block2, %Block2 addrspace(1)* @block2, i32 0, i32 0
+  %tmp1 = load i32, i32 addrspace(1)* %tmp
+  store i32 %tmp1, i32* %i
+  %tmp2 = load i32, i32* %i
+  %tmp3 = sitofp i32 %tmp2 to float
+  %tmp4 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 0
+  store float %tmp3, float* %tmp4
+  %tmp5 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 1
+  store float 2.000000e+00, float* %tmp5
+  %tmp6 = load [2 x float], [2 x float]* %f1
+  %tmp7 = getelementptr %Block2, %Block2 addrspace(1)* @block2, i32 0, i32 1
+  store [2 x float] %tmp6, [2 x float] addrspace(1)* %tmp7
+  %tmp8 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
+  store <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <4 x float> addrspace(1)* %tmp8
+  ret void
+}
+
+; OPT-LABEL: @promote_load_from_store_aggr(
+; OPT: store [2 x float] %tmp3, [2 x float]* %f1
+
+%Block3 = type { [2 x float], i32 }
+@block3 = external addrspace(1) global %Block3
+
+define amdgpu_vs void @promote_load_from_store_aggr() #0 {
+  %i = alloca i32
+  %f1 = alloca [2 x float]
+  %tmp = getelementptr %Block3, %Block3 addrspace(1)* @block3, i32 0, i32 1
+  %tmp1 = load i32, i32 addrspace(1)* %tmp
+  store i32 %tmp1, i32* %i
+  %tmp2 = getelementptr %Block3, %Block3 addrspace(1)* @block3, i32 0, i32 0
+  %tmp3 = load [2 x float], [2 x float] addrspace(1)* %tmp2
+  store [2 x float] %tmp3, [2 x float]* %f1
+  %tmp4 = load i32, i32* %i
+  %tmp5 = getelementptr [2 x float], [2 x float]* %f1, i32 0, i32 %tmp4
+  %tmp6 = load float, float* %tmp5
+  %tmp7 = alloca <4 x float>
+  %tmp8 = load <4 x float>, <4 x float>* %tmp7
+  %tmp9 = insertelement <4 x float> %tmp8, float %tmp6, i32 0
+  %tmp10 = insertelement <4 x float> %tmp9, float %tmp6, i32 1
+  %tmp11 = insertelement <4 x float> %tmp10, float %tmp6, i32 2
+  %tmp12 = insertelement <4 x float> %tmp11, float %tmp6, i32 3
+  %tmp13 = getelementptr %gl_PerVertex, %gl_PerVertex addrspace(1)* @pv, i32 0, i32 0
+  store <4 x float> %tmp12, <4 x float> addrspace(1)* %tmp13
+  ret void
+}
+
+; OPT-LABEL: @promote_double_aggr(
+; OPT: store [2 x double] %tmp5, [2 x double]* %s
+
+@tmp_g = external addrspace(1) global { [4 x double], <2 x double>, <3 x double>, <4 x double> }
+@frag_color = external addrspace(1) global <4 x float>
+
+define amdgpu_ps void @promote_double_aggr() #0 {
+  %s = alloca [2 x double]
+  %tmp = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, { [4 x double], <2 x double>, <3 x double>, <4 x double> } addrspace(1)* @tmp_g, i32 0, i32 0, i32 0
+  %tmp1 = load double, double addrspace(1)* %tmp
+  %tmp2 = getelementptr { [4 x double], <2 x double>, <3 x double>, <4 x double> }, { [4 x double], <2 x double>, <3 x double>, <4 x double> } addrspace(1)* @tmp_g, i32 0, i32 0, i32 1
+  %tmp3 = load double, double addrspace(1)* %tmp2
+  %tmp4 = insertvalue [2 x double] undef, double %tmp1, 0
+  %tmp5 = insertvalue [2 x double] %tmp4, double %tmp3, 1
+  store [2 x double] %tmp5, [2 x double]* %s
+  %tmp6 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
+  %tmp7 = load double, double* %tmp6
+  %tmp8 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
+  %tmp9 = load double, double* %tmp8
+  %tmp10 = fadd double %tmp7, %tmp9
+  %tmp11 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 0
+  store double %tmp10, double* %tmp11
+  %tmp12 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 0
+  %tmp13 = load double, double* %tmp12
+  %tmp14 = getelementptr [2 x double], [2 x double]* %s, i32 0, i32 1
+  %tmp15 = load double, double* %tmp14
+  %tmp16 = fadd double %tmp13, %tmp15
+  %tmp17 = fptrunc double %tmp16 to float
+  %tmp18 = insertelement <4 x float> undef, float %tmp17, i32 0
+  %tmp19 = insertelement <4 x float> %tmp18, float %tmp17, i32 1
+  %tmp20 = insertelement <4 x float> %tmp19, float %tmp17, i32 2
+  %tmp21 = insertelement <4 x float> %tmp20, float %tmp17, i32 3
+  store <4 x float> %tmp21, <4 x float> addrspace(1)* @frag_color
+  ret void
+}
+
+; Don't crash on a type that isn't a valid vector element.
+; OPT-LABEL: @alloca_struct(
+define amdgpu_kernel void @alloca_struct() #0 {
+entry:
+  %alloca = alloca [2 x %struct], align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll b/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll
index a5eb92de9e4b5..b440efc808282 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-bitcast-function.ll
@@ -1,19 +1,23 @@
 ; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck %s
 
+; FIXME: Error is misleading because it's not an indirect call.
+
+; CHECK: error: <unknown>:0:0: in function crash_call_constexpr_cast void (): unsupported indirect call to function foo
+
 ; Make sure that AMDGPUPromoteAlloca doesn't crash if the called
 ; function is a constantexpr cast of a function.
 
 declare void @foo(float*) #0
 declare void @foo.varargs(...) #0
 
-; CHECK: in function crash_call_constexpr_cast{{.*}}: unsupported call to function foo
+; XCHECK: in function crash_call_constexpr_cast{{.*}}: unsupported call to function foo
 define amdgpu_kernel void @crash_call_constexpr_cast() #0 {
   %alloca = alloca i32
   call void bitcast (void (float*)* @foo to void (i32*)*)(i32* %alloca) #0
   ret void
 }
 
-; CHECK: in function crash_call_constexpr_cast{{.*}}: unsupported call to function foo.varargs
+; XCHECK: in function crash_call_constexpr_cast{{.*}}: unsupported call to function foo.varargs
 define amdgpu_kernel void @crash_call_constexpr_cast_varargs() #0 {
   %alloca = alloca i32
   call void bitcast (void (...)* @foo.varargs to void (i32*)*)(i32* %alloca) #0
diff --git a/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll b/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll
index a95e9f828b615..515cf8140c331 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-calling-conv.ll
@@ -1,11 +1,11 @@
 ; RUN: opt -S -mtriple=amdgcn-unknown-unknown -amdgpu-promote-alloca < %s | FileCheck -check-prefix=IR %s
-; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=ASM %s
+; RUN: llc -march=amdgcn -mcpu=fiji < %s | FileCheck -check-prefix=ASM %s
 
 ; IR-LABEL: define amdgpu_vs void @promote_alloca_shaders(i32 addrspace(1)* inreg %out, i32 addrspace(1)* inreg %in) #0 {
 ; IR: alloca [5 x i32]
-; ASM-LABEL: {{^}}promote_alloca_shaders:
-; ASM: ; LDSByteSize: 0 bytes/workgroup (compile time only)
 
+; ASM-LABEL: {{^}}promote_alloca_shaders:
+; ASM: ; ScratchSize: 24
 define amdgpu_vs void @promote_alloca_shaders(i32 addrspace(1)* inreg %out, i32 addrspace(1)* inreg %in) #0 {
 entry:
   %stack = alloca [5 x i32], align 4
@@ -29,7 +29,10 @@ entry:
 ; OPT-LABEL: @promote_to_vector_call_c(
 ; OPT-NOT: alloca
 ; OPT: extractelement <2 x i32> %{{[0-9]+}}, i32 %in
+
+; ASM-LABEL: {{^}}promote_to_vector_call_c:
 ; ASM-NOT: LDSByteSize
+; ASM: ; ScratchSize: 0
 define void @promote_to_vector_call_c(i32 addrspace(1)* %out, i32 %in) #0 {
 entry:
   %tmp = alloca [2 x i32]
@@ -47,8 +50,11 @@ entry:
 
 ; OPT-LABEL: @no_promote_to_lds_c(
 ; OPT: alloca
+
+; ASM-LABEL: {{^}}no_promote_to_lds_c:
 ; ASM-NOT: LDSByteSize
-define void @no_promote_to_lds(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #0 {
+; ASM: ; ScratchSize: 24
+define void @no_promote_to_lds_c(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #0 {
 entry:
   %stack = alloca [5 x i32], align 4
   %0 = load i32, i32 addrspace(1)* %in, align 4
@@ -68,6 +74,26 @@ entry:
   ret void
 }
 
+declare i32 @foo(i32*) #0
+
+; ASM-LABEL: {{^}}call_private:
+; ASM: buffer_store_dword
+; ASM: buffer_store_dword
+; ASM: s_swappc_b64
+; ASM: ScratchSize: 16396
+define amdgpu_kernel void @call_private(i32 addrspace(1)* %out, i32 %in) #0 {
+entry:
+  %tmp = alloca [2 x i32]
+  %tmp1 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 0
+  %tmp2 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 1
+  store i32 0, i32* %tmp1
+  store i32 1, i32* %tmp2
+  %tmp3 = getelementptr [2 x i32], [2 x i32]* %tmp, i32 0, i32 %in
+  %val = call i32 @foo(i32* %tmp3)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
 attributes #0 = { nounwind "amdgpu-max-work-group-size"="64" }
diff --git a/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll b/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
index 776b151e30170..768d520ddd535 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll
@@ -1,22 +1,23 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+target datalayout = "A5"
 
-declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) #0
-declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) #0
-declare i8* @llvm.invariant.group.barrier(i8*) #1
+declare {}* @llvm.invariant.start.p5i8(i64, i8 addrspace(5)* nocapture) #0
+declare void @llvm.invariant.end.p5i8({}*, i64, i8 addrspace(5)* nocapture) #0
+declare i8 addrspace(5)* @llvm.invariant.group.barrier.p5i8(i8 addrspace(5)*) #1
 
 ; GCN-LABEL: {{^}}use_invariant_promotable_lds:
 ; GCN: buffer_load_dword
 ; GCN: ds_write_b32
 define amdgpu_kernel void @use_invariant_promotable_lds(i32 addrspace(1)* %arg) #2 {
 bb:
-  %tmp = alloca i32, align 4
-  %tmp1 = bitcast i32* %tmp to i8*
+  %tmp = alloca i32, align 4, addrspace(5)
+  %tmp1 = bitcast i32 addrspace(5)* %tmp to i8 addrspace(5)*
   %tmp2 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1
   %tmp3 = load i32, i32 addrspace(1)* %tmp2
-  store i32 %tmp3, i32* %tmp
-  %tmp4 = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %tmp1) #0
-  call void @llvm.invariant.end.p0i8({}* %tmp4, i64 4, i8* %tmp1) #0
-  %tmp5 = call i8* @llvm.invariant.group.barrier(i8* %tmp1) #1
+  store i32 %tmp3, i32 addrspace(5)* %tmp
+  %tmp4 = call {}* @llvm.invariant.start.p5i8(i64 4, i8 addrspace(5)* %tmp1) #0
+  call void @llvm.invariant.end.p5i8({}* %tmp4, i64 4, i8 addrspace(5)* %tmp1) #0
+  %tmp5 = call i8 addrspace(5)* @llvm.invariant.group.barrier.p5i8(i8 addrspace(5)* %tmp1) #1
   ret void
 }
 
diff --git a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
index 7a4a451ff3603..7343dd6bbdad0 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -amdgpu-promote-alloca < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck %s
 
 declare void @llvm.memcpy.p0i8.p1i8.i32(i8* nocapture, i8 addrspace(1)* nocapture, i32, i32, i1) #0
 declare void @llvm.memcpy.p1i8.p0i8.i32(i8 addrspace(1)* nocapture, i8* nocapture, i32, i32, i1) #0
diff --git a/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll b/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
index 9f22f2071797e..653e259e155f0 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll
@@ -1,5 +1,5 @@
-; RUN: llc -O0 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mattr=+promote-alloca < %s | FileCheck -check-prefix=NOOPTS -check-prefix=ALL %s
-; RUN: llc -O1 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mattr=+promote-alloca < %s | FileCheck -check-prefix=OPTS -check-prefix=ALL %s
+; RUN: llc -O0 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -mattr=+promote-alloca < %s | FileCheck -check-prefix=NOOPTS -check-prefix=ALL %s
+; RUN: llc -O1 -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -mattr=+promote-alloca < %s | FileCheck -check-prefix=OPTS -check-prefix=ALL %s
 
 ; ALL-LABEL: {{^}}promote_alloca_i32_array_array:
 ; NOOPTS: workgroup_group_segment_byte_size = 0{{$}}
diff --git a/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll b/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
index bf3bc493a4b86..137b41df55cfe 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri < %s | FileCheck -check-prefix=GCN %s
 
 ; This shows that the amount of LDS estimate is sensitive to the order
 ; of the LDS globals.
diff --git a/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll b/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
index 55c2229fb6bdb..ebeed0dd4435b 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll
@@ -130,4 +130,4 @@ bb:
 }
 
 attributes #0 = { norecurse nounwind "amdgpu-waves-per-eu"="1,1" }
-attributes #1 = { norecurse nounwind }
-\ No newline at end of file
+attributes #1 = { norecurse nounwind }
diff --git a/test/CodeGen/AMDGPU/r600.amdgpu-alias-analysis.ll b/test/CodeGen/AMDGPU/r600.amdgpu-alias-analysis.ll
index 8956d113e8b50..b2d6139ca3dc1 100644
--- a/test/CodeGen/AMDGPU/r600.amdgpu-alias-analysis.ll
+++ b/test/CodeGen/AMDGPU/r600.amdgpu-alias-analysis.ll
@@ -1,7 +1,7 @@
-; RUN: opt -mtriple=r600-- -O3 -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -mtriple=r600---amdgiz -O3 -aa-eval -print-all-alias-modref-info -disable-output < %s 2>&1 | FileCheck %s
 
-; CHECK: NoAlias:      i8 addrspace(7)* %p1, i8* %p
+; CHECK: NoAlias: i8 addrspace(5)* %p, i8 addrspace(7)* %p1
 
-define amdgpu_kernel void @test(i8* %p, i8 addrspace(7)* %p1) {
+define amdgpu_kernel void @test(i8 addrspace(5)* %p, i8 addrspace(7)* %p1) {
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/readlane_exec0.mir b/test/CodeGen/AMDGPU/readlane_exec0.mir
new file mode 100644
index 0000000000000..b6d58d74ebddb
--- /dev/null
+++ b/test/CodeGen/AMDGPU/readlane_exec0.mir
@@ -0,0 +1,32 @@
+# RUN: llc -o - %s -march=amdgcn -mcpu=fiji  -run-pass=si-insert-skips -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: readlane_exec0
+# GCN: bb.0
+# GCN: S_CBRANCH_EXECZ %bb.2
+
+---
+name: readlane_exec0
+
+body:       |
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: %vgpr1_vgpr2:0x00000001, %vgpr2_vgpr3:0x00000003
+
+    %vgpr4 = V_AND_B32_e32 1, %vgpr1, implicit %exec
+    V_CMP_EQ_U32_e32 1, killed %vgpr4, implicit-def %vcc, implicit %exec
+    %sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 killed %vcc, implicit-def %exec, implicit-def %scc, implicit %exec
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
+
+  bb.1:
+
+   %sgpr10 = V_READFIRSTLANE_B32 %vgpr2, implicit %exec
+   %sgpr11 = V_READFIRSTLANE_B32 %vgpr3, implicit %exec
+   %sgpr10 = S_LOAD_DWORD_IMM killed %sgpr10_sgpr11, 0, 0
+   S_WAITCNT 127
+   %vgpr0 = V_XOR_B32_e32 killed %sgpr10, killed %vgpr0, implicit %exec
+
+  bb.2:
+
+    %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc
+...
diff --git a/test/CodeGen/AMDGPU/reduce-saveexec.mir b/test/CodeGen/AMDGPU/reduce-saveexec.mir
new file mode 100644
index 0000000000000..6f6b0deb2b98f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/reduce-saveexec.mir
@@ -0,0 +1,147 @@
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-optimize-exec-masking %s -o - | FileCheck -check-prefix=GCN %s
+
+---
+# GCN-LABEL: name: reduce_and_saveexec
+# GCN:      %exec = S_AND_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_and_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_AND_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_and_saveexec_commuted
+# GCN:      %exec = S_AND_B64 killed %vcc, %exec
+# GCN-NEXT: S_ENDPGM
+name: reduce_and_saveexec_commuted
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_AND_B64 killed %vcc, %exec, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_and_saveexec_liveout
+# GCN:      %sgpr0_sgpr1 = S_AND_B64 %exec, killed %vcc
+# GCN-NEXT: %exec = COPY
+name: reduce_and_saveexec_liveout
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_AND_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: and_saveexec
+# GCN:      %sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 %vcc
+# GCN-NEXT: S_ENDPGM
+name: and_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = COPY %exec
+    %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc
+    %exec = S_MOV_B64_term %sgpr2_sgpr3
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_or_saveexec
+# GCN:      %exec = S_OR_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_or_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_OR_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_xor_saveexec
+# GCN:      %exec = S_XOR_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_xor_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_XOR_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_andn2_saveexec
+# GCN:      %exec = S_ANDN2_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_andn2_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_ANDN2_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_orn2_saveexec
+# GCN:      %exec = S_ORN2_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_orn2_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_ORN2_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_nand_saveexec
+# GCN:      %exec = S_NAND_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_nand_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_NAND_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_nor_saveexec
+# GCN:      %exec = S_NOR_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_nor_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_NOR_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
+# GCN-LABEL: name: reduce_xnor_saveexec
+# GCN:      %exec = S_XNOR_B64 %exec, killed %vcc
+# GCN-NEXT: S_ENDPGM
+name: reduce_xnor_saveexec
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %vcc = IMPLICIT_DEF
+    %sgpr0_sgpr1 = S_XNOR_B64 %exec, killed %vcc, implicit-def %scc
+    %exec = COPY killed %sgpr0_sgpr1
+    S_ENDPGM
+...
+---
diff --git a/test/CodeGen/AMDGPU/regcoal-subrange-join.mir b/test/CodeGen/AMDGPU/regcoal-subrange-join.mir
index bac348aaed709..a03135348ce19 100644
--- a/test/CodeGen/AMDGPU/regcoal-subrange-join.mir
+++ b/test/CodeGen/AMDGPU/regcoal-subrange-join.mir
@@ -4,10 +4,10 @@
 # This test will provoke a subrange join (see annotations below) during simple register coalescing
 # Without a fix for PR33524 this causes an unreachable in SubRange Join
 #
-# GCN-DAG: undef %[[REG0:[0-9]+]].sub0 = COPY %sgpr5
-# GCN-DAG: undef %[[REG1:[0-9]+]].sub0 = COPY %sgpr2
-# GCN-DAG: %[[REG0]].sub1 = S_MOV_B32 1
-# GCN-DAG: %[[REG1]].sub1 = S_MOV_B32 1
+# GCN-DAG: undef %[[REG0:[0-9]+]].sub0:sgpr_64 = COPY %sgpr5
+# GCN-DAG: undef %[[REG1:[0-9]+]].sub0:sgpr_64 = COPY %sgpr2
+# GCN-DAG: %[[REG0]].sub1:sgpr_64 = S_MOV_B32 1
+# GCN-DAG: %[[REG1]].sub1:sgpr_64 = S_MOV_B32 1
 
 --- |
   define amdgpu_vs void @regcoal-subrange-join(i32 inreg %arg, i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3, i32 inreg %arg4, i32 inreg %arg5, i32 %arg6) local_unnamed_addr #0 {
diff --git a/test/CodeGen/AMDGPU/regcoalesce-dbg.mir b/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
index ecf94b5772ffc..69538d8b7382f 100644
--- a/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
+++ b/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
@@ -4,12 +4,12 @@
 # LIS->getInstructionIndex with a DBG_VALUE instruction, which does not have
 # a slot index.
 
-# CHECK: %13.sub2 = S_MOV_B32 0
+# CHECK: %13.sub2:sgpr_128 = S_MOV_B32 0
 # CHECK: DBG_VALUE{{.*}}debug-use %13.sub2
 
 --- |
   define amdgpu_kernel void @test(i32 addrspace(1)* %out) { ret void }
-  
+
   !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !4, producer: "llvm", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4)
   !1 = !DILocalVariable(name: "a", scope: !2, file: !4, line: 126, type: !6)
   !2 = distinct !DISubprogram(name: "test", scope: !4, file: !4, line: 1, type: !3, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, variables: !5)
@@ -25,7 +25,7 @@
 ---
 name:            test
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: sgpr_64 }
   - { id: 1, class: sreg_32_xm0 }
   - { id: 2, class: sgpr_32 }
@@ -47,13 +47,13 @@ registers:
   - { id: 18, class: vgpr_32 }
   - { id: 19, class: vreg_64 }
   - { id: 20, class: vreg_64 }
-liveins:         
+liveins:
   - { reg: '%sgpr0_sgpr1', virtual-reg: '%0' }
   - { reg: '%vgpr0', virtual-reg: '%3' }
 body:             |
   bb.0:
     liveins: %sgpr0_sgpr1, %vgpr0
-  
+
     %3 = COPY killed %vgpr0
     %0 = COPY killed %sgpr0_sgpr1
     %4 = S_LOAD_DWORDX2_IMM %0, 9, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
@@ -63,7 +63,7 @@ body:             |
     %19.sub1 = COPY killed %18
     %10 = S_MOV_B32 61440
     %11 = S_MOV_B32 0
-    DBG_VALUE debug-use %11, debug-use _, !1, !8, debug-location !9
+    DBG_VALUE debug-use %11, debug-use %noreg, !1, !8, debug-location !9
     undef %12.sub0 = COPY killed %11
     %12.sub1 = COPY killed %10
     undef %13.sub0_sub1 = COPY killed %4
diff --git a/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir b/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
index a52b80ba86e59..08b3ecf8dbac3 100644
--- a/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
+++ b/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
@@ -2,7 +2,7 @@
 ---
 
 # GCN-LABEL: name: mac_invalid_operands
-# GCN: undef %18.sub0 = V_MAC_F32_e32 undef %3, undef %9, undef %18.sub0, implicit %exec
+# GCN: undef %18.sub0:vreg_128 = V_MAC_F32_e32 undef %3:vgpr_32, undef %9:vgpr_32, undef %18.sub0, implicit %exec
 
 name:            mac_invalid_operands
 alignment:       0
@@ -73,13 +73,13 @@ body:             |
 
 # GCN-LABEL: name: vreg_does_not_dominate
 
-# GCN: undef %8.sub1 = V_MAC_F32_e32 undef %2, undef %1, undef %8.sub1, implicit %exec
-# GCN: undef %7.sub0 = V_MOV_B32_e32 0, implicit %exec
-# GCN: undef %9.sub2 = COPY %7.sub0
+# GCN: undef %8.sub1:vreg_128 = V_MAC_F32_e32 undef %2:vgpr_32, undef %1:vgpr_32, undef %8.sub1, implicit %exec
+# GCN: undef %7.sub0:vreg_128 = V_MOV_B32_e32 0, implicit %exec
+# GCN: undef %9.sub2:vreg_128 = COPY %7.sub0
 
-# GCN: undef %6.sub3 = V_ADD_F32_e32 undef %3, undef %3, implicit %exec
-# GCN: undef %7.sub0 = V_ADD_F32_e64 0, 0, 0, 0, 0, 0, implicit %exec
-# GCN: %8.sub1 = V_ADD_F32_e32 %8.sub1, %8.sub1, implicit %exec
+# GCN: undef %6.sub3:vreg_128 = V_ADD_F32_e32 undef %3:vgpr_32, undef %3:vgpr_32, implicit %exec
+# GCN: undef %7.sub0:vreg_128 = V_ADD_F32_e64 0, 0, 0, 0, 0, 0, implicit %exec
+# GCN: %8.sub1:vreg_128 = V_ADD_F32_e32 %8.sub1, %8.sub1, implicit %exec
 
 # GCN: BUFFER_STORE_DWORD_OFFEN %6.sub3, %0,
 # GCN: BUFFER_STORE_DWORD_OFFEN %9.sub2, %0,
@@ -137,8 +137,8 @@ body:             |
 
 # GCN-LABEL: name: inf_loop_tied_operand
 # GCN: bb.0:
-# GCN-NEXT: undef %2.sub0 = V_MAC_F32_e32 1073741824, undef %0, undef %2.sub0, implicit %exec
-# GCN-NEXT: dead undef %3.sub1 = COPY %2.sub0
+# GCN-NEXT: undef %2.sub0:vreg_128 = V_MAC_F32_e32 1073741824, undef %0:vgpr_32, undef %2.sub0, implicit %exec
+# GCN-NEXT: dead undef %3.sub1:vreg_128 = COPY %2.sub0
 
 name:            inf_loop_tied_operand
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/ret_jump.ll b/test/CodeGen/AMDGPU/ret_jump.ll
index 1acae60f30579..f87e8cbea4fc2 100644
--- a/test/CodeGen/AMDGPU/ret_jump.ll
+++ b/test/CodeGen/AMDGPU/ret_jump.ll
@@ -11,7 +11,6 @@
 ; GCN-NEXT: ; %else
 
 ; GCN: s_and_saveexec_b64 [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN-NEXT: s_xor_b64 [[XOR_EXEC:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE_EXEC]]
 ; GCN-NEXT: ; mask branch [[FLOW:BB[0-9]+_[0-9]+]]
 
 ; GCN: BB{{[0-9]+_[0-9]+}}: ; %unreachable.bb
@@ -58,9 +57,8 @@ ret.bb:                                          ; preds = %else, %main_body
 ; GCN-LABEL: {{^}}uniform_br_nontrivial_ret_divergent_br_nontrivial_unreachable:
 ; GCN: s_cbranch_vccnz [[RET_BB:BB[0-9]+_[0-9]+]]
 
-; GCN: ; BB#{{[0-9]+}}: ; %else
+; GCN: ; %bb.{{[0-9]+}}: ; %else
 ; GCN: s_and_saveexec_b64 [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN-NEXT: s_xor_b64 [[XOR_EXEC:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE_EXEC]]
 ; GCN-NEXT: ; mask branch [[FLOW1:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT:  ; %unreachable.bb
diff --git a/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll b/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll
new file mode 100644
index 0000000000000..e0aed8aa33114
--- /dev/null
+++ b/test/CodeGen/AMDGPU/rewrite-out-arguments-address-space.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-any-address-space-out-arguments -amdgpu-rewrite-out-arguments < %s | FileCheck %s
+
+; CHECK: %void_one_out_non_private_arg_i32_1_use = type { i32 }
+; CHECK: %bitcast_pointer_as1 = type { <3 x i32> }
+
+; CHECK-LABEL: define private %void_one_out_non_private_arg_i32_1_use @void_one_out_non_private_arg_i32_1_use.body(i32 addrspace(1)* %val) #0 {
+; CHECK-NEXT: ret %void_one_out_non_private_arg_i32_1_use zeroinitializer
+
+; CHECK-LABEL: define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)*) #1 {
+; CHECK-NEXT: %2 = call %void_one_out_non_private_arg_i32_1_use @void_one_out_non_private_arg_i32_1_use.body(i32 addrspace(1)* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_non_private_arg_i32_1_use %2, 0
+; CHECK-NEXT: store i32 %3, i32 addrspace(1)* %0, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %val) #0 {
+  store i32 0, i32 addrspace(1)* %val
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_pointer_as1 @bitcast_pointer_as1.body(<3 x i32> addrspace(1)* %out) #0 {
+; CHECK-NEXT: %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+; CHECK-NEXT: %bitcast = bitcast <3 x i32> addrspace(1)* %out to <4 x i32> addrspace(1)*
+; CHECK-NEXT: %1 = shufflevector <4 x i32> %load, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_as1 undef, <3 x i32> %1, 0
+; CHECK-NEXT: ret %bitcast_pointer_as1 %2
+
+; CHECK-LABEL: define void @bitcast_pointer_as1(<3 x i32> addrspace(1)*) #1 {
+; CHECK-NEXT: %2 = call %bitcast_pointer_as1 @bitcast_pointer_as1.body(<3 x i32> addrspace(1)* undef)
+define void @bitcast_pointer_as1(<3 x i32> addrspace(1)* %out) #0 {
+  %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+  %bitcast = bitcast <3 x i32> addrspace(1)* %out to <4 x i32> addrspace(1)*
+  store <4 x i32> %load, <4 x i32> addrspace(1)* %bitcast
+  ret void
+}
+
+; CHECK: attributes #0 = { nounwind }
+; CHECK: attributes #1 = { alwaysinline nounwind }
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/rewrite-out-arguments.ll b/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
new file mode 100644
index 0000000000000..ad092fab64023
--- /dev/null
+++ b/test/CodeGen/AMDGPU/rewrite-out-arguments.ll
@@ -0,0 +1,823 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -amdgpu-rewrite-out-arguments < %s | FileCheck %s
+
+; CHECK: %void_one_out_arg_i32_1_use = type { i32 }
+; CHECK: %void_one_out_arg_i32_1_use_align = type { i32 }
+; CHECK: %void_one_out_arg_i32_2_use = type { i32 }
+; CHECK: %void_one_out_arg_i32_2_stores = type { i32 }
+; CHECK: %void_one_out_arg_i32_2_stores_clobber = type { i32 }
+; CHECK: %void_one_out_arg_i32_pre_call_may_clobber = type { i32 }
+; CHECK: %void_one_out_arg_v2i32_1_use = type { <2 x i32> }
+; CHECK: %void_one_out_arg_struct_1_use = type { %struct }
+; CHECK: %struct = type { i32, i8, float }
+; CHECK: %i32_one_out_arg_i32_1_use = type { i32, i32 }
+; CHECK: %unused_different_type = type { float }
+; CHECK: %multiple_same_return_noalias = type { i32, i32 }
+; CHECK: %multiple_same_return_mayalias = type { i32, i32 }
+; CHECK: %multiple_same_return_mayalias_order = type { i32, i32 }
+; CHECK: %i1_one_out_arg_i32_1_use = type { i1, i32 }
+; CHECK: %i1_zeroext_one_out_arg_i32_1_use = type { i1, i32 }
+; CHECK: %i1_signext_one_out_arg_i32_1_use = type { i1, i32 }
+; CHECK: %p1i32_noalias_one_out_arg_i32_1_use = type { i32 addrspace(1)*, i32 }
+; CHECK: %func_ptr_type = type { void ()* }
+; CHECK: %bitcast_func_ptr_type = type { void ()* }
+; CHECK: %out_arg_small_array = type { [4 x i32] }
+; CHECK: %num_regs_reach_limit = type { [15 x i32], i32 }
+; CHECK: %num_regs_reach_limit_leftover = type { [15 x i32], i32, i32 }
+; CHECK: %preserve_debug_info = type { i32 }
+; CHECK: %preserve_metadata = type { i32 }
+; CHECK: %bitcast_pointer_v4i32_v3i32 = type { <3 x i32> }
+; CHECK: %bitcast_pointer_v4i32_v3f32 = type { <3 x float> }
+; CHECK: %bitcast_pointer_i32_f32 = type { float }
+; CHECK: %bitcast_struct_v3f32_v3f32 = type { %struct.v3f32 }
+; CHECK: %struct.v3f32 = type { <3 x float> }
+; CHECK: %bitcast_struct_v3f32_v3i32 = type { %struct.v3f32 }
+; CHECK: %bitcast_struct_v4f32_v4f32 = type { %struct.v4f32 }
+; CHECK: %struct.v4f32 = type { <4 x float> }
+; CHECK: %bitcast_struct_v3f32_v4i32 = type { %struct.v3f32 }
+; CHECK: %bitcast_struct_v4f32_v3f32 = type { %struct.v4f32 }
+; CHECK: %struct.v3f32.f32 = type { <3 x float>, float }
+; CHECK: %bitcast_struct_i128_v4f32 = type { %struct.i128 }
+; CHECK: %struct.i128 = type { i128 }
+; CHECK: %multi_return_bitcast_struct_v3f32_v3f32 = type { %struct.v3f32 }
+
+; CHECK-LABEL: define void @no_ret_blocks() #0 {
+; CHECK-NEXT: unreachable
+define void @no_ret_blocks() #0 {
+  unreachable
+}
+
+; CHECK-LABEL: @void_one_out_arg_i32_no_use(
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_no_use(i32* %val) #0 {
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_byval_arg(
+; CHECK-NEXT: store i32 0, i32* %val
+; CHECK-NEXT: ret void
+define void @skip_byval_arg(i32* byval %val) #0 {
+  store i32 0, i32* %val
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_optnone(
+; CHECK-NEXT: store i32 0, i32* %val
+; CHECK-NEXT: ret void
+define void @skip_optnone(i32* byval %val) #1 {
+  store i32 0, i32* %val
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_volatile(
+; CHECK-NEXT: store volatile i32 0, i32* %val
+; CHECK-NEXT: ret void
+define void @skip_volatile(i32* byval %val) #0 {
+  store volatile i32 0, i32* %val
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_atomic(
+; CHECK-NEXT: store atomic i32 0, i32* %val
+; CHECK-NEXT: ret void
+define void @skip_atomic(i32* byval %val) #0 {
+  store atomic i32 0, i32* %val seq_cst, align 4
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_store_pointer_val(
+; CHECK-NEXT: store i32* %val, i32** undef
+; CHECK-NEXT: ret void
+define void @skip_store_pointer_val(i32* %val) #0 {
+  store i32* %val, i32** undef
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @skip_store_gep(
+; CHECK-NEXT: %gep = getelementptr inbounds i32, i32* %val, i32 1
+; CHECK-NEXT: store i32 0, i32* %gep
+; CHECK-NEXT: ret void
+define void @skip_store_gep(i32* %val) #0 {
+  %gep = getelementptr inbounds i32, i32* %val, i32 1
+  store i32 0, i32* %gep
+  ret void
+}
+
+; CHECK-LABEL: define void @skip_sret(i32* sret %sret, i32* %out) #0 {
+; CHECK-NEXT: store
+; CHECK-NEXT: store
+; CHECK-NEXT: ret void
+define void @skip_sret(i32* sret %sret, i32* %out) #0 {
+  store i32 1, i32* %sret
+  store i32 0, i32* %out
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_1_use @void_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %void_one_out_arg_i32_1_use zeroinitializer
+
+; CHECK-LABEL: @void_one_out_arg_i32_1_use(
+; CHECK-NEXT: %2 = call %void_one_out_arg_i32_1_use @void_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_1_use %2, 0
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 0, i32* %val
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_1_use_align @void_one_out_arg_i32_1_use_align.body(i32* align 8 %val) #0 {
+; CHECK-NEXT: ret %void_one_out_arg_i32_1_use_align zeroinitializer
+
+; CHECK-LABEL: @void_one_out_arg_i32_1_use_align(
+; CHECK-NEXT: %2 = call %void_one_out_arg_i32_1_use_align @void_one_out_arg_i32_1_use_align.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_1_use_align %2, 0
+; CHECK-NEXT: store i32 %3, i32* %0, align 8
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_1_use_align(i32* align 8 %val) #0 {
+  store i32 0, i32* %val, align 8
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_2_use @void_one_out_arg_i32_2_use.body(i1 %arg0, i32* %val) #0 {
+; CHECK: br i1 %arg0, label %ret0, label %ret1
+
+; CHECK: ret0:
+; CHECK-NEXT: ret %void_one_out_arg_i32_2_use zeroinitializer
+
+; CHECK: ret1:
+; CHECK-NEXT: ret %void_one_out_arg_i32_2_use { i32 9 }
+
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_use(i1, i32*) #2 {
+; CHECK-NEXT: %3 = call %void_one_out_arg_i32_2_use @void_one_out_arg_i32_2_use.body(i1 %0, i32* undef)
+; CHECK-NEXT: %4 = extractvalue %void_one_out_arg_i32_2_use %3, 0
+; CHECK-NEXT: store i32 %4, i32* %1, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_2_use(i1 %arg0, i32* %val) #0 {
+  br i1 %arg0, label %ret0, label %ret1
+
+ret0:
+  store i32 0, i32* %val
+  ret void
+
+ret1:
+  store i32 9, i32* %val
+  ret void
+}
+
+declare void @may.clobber()
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_2_stores @void_one_out_arg_i32_2_stores.body(i32* %val) #0 {
+; CHECK-NEXT: store i32 0, i32* %val
+; CHECK-NEXT: ret %void_one_out_arg_i32_2_stores { i32 1 }
+
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores(i32*) #2 {
+; CHECK-NEXT: %2 = call %void_one_out_arg_i32_2_stores @void_one_out_arg_i32_2_stores.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_2_stores %2, 0
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+define void @void_one_out_arg_i32_2_stores(i32* %val) #0 {
+  store i32 0, i32* %val
+  store i32 1, i32* %val
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_2_stores_clobber @void_one_out_arg_i32_2_stores_clobber.body(i32* %val) #0 {
+; CHECK-NEXT: store i32 0, i32* %val
+; CHECK-NEXT: call void @may.clobber()
+; CHECK-NEXT: ret %void_one_out_arg_i32_2_stores_clobber { i32 1 }
+
+; CHECK-LABEL: define void @void_one_out_arg_i32_2_stores_clobber(i32*) #2 {
+; CHECK-NEXT: %2 = call %void_one_out_arg_i32_2_stores_clobber @void_one_out_arg_i32_2_stores_clobber.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_2_stores_clobber %2, 0
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_2_stores_clobber(i32* %val) #0 {
+  store i32 0, i32* %val
+  call void @may.clobber()
+  store i32 1, i32* %val
+  ret void
+}
+
+; CHECK-NOT: define
+
+; CHECK-LABEL: define void @void_one_out_arg_i32_call_may_clobber(i32* %val) #0 {
+; CHECK-NEXT: store i32 0, i32* %val
+; CHECK-NEXT: call void @may.clobber()
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_call_may_clobber(i32* %val) #0 {
+  store i32 0, i32* %val
+  call void @may.clobber()
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_i32_pre_call_may_clobber @void_one_out_arg_i32_pre_call_may_clobber.body(i32* %val) #0 {
+; CHECK-NEXT: call void @may.clobber()
+; CHECK-NEXT: ret %void_one_out_arg_i32_pre_call_may_clobber zeroinitializer
+
+; CHECK-LABEL: @void_one_out_arg_i32_pre_call_may_clobber(i32*) #2 {
+; CHECK-NEXT: %2 = call %void_one_out_arg_i32_pre_call_may_clobber @void_one_out_arg_i32_pre_call_may_clobber.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_i32_pre_call_may_clobber %2, 0
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_pre_call_may_clobber(i32* %val) #0 {
+  call void @may.clobber()
+  store i32 0, i32* %val
+  ret void
+}
+
+; CHECK-LABEL: define void @void_one_out_arg_i32_reload(i32* %val) #0 {
+; CHECK: store i32 0, i32* %val
+; CHECK: %load = load i32, i32* %val, align 4
+; CHECK: ret void
+define void @void_one_out_arg_i32_reload(i32* %val) #0 {
+  store i32 0, i32* %val
+  %load = load i32, i32* %val, align 4
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @void_one_out_arg_i32_store_in_different_block(
+; CHECK-NEXT: %load = load i32, i32 addrspace(1)* undef
+; CHECK-NEXT: store i32 0, i32* %out
+; CHECK-NEXT: br label %ret
+; CHECK: ret:
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_i32_store_in_different_block(i32* %out) #0 {
+  %load = load i32, i32 addrspace(1)* undef
+  store i32 0, i32* %out
+  br label %ret
+
+ret:
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @unused_out_arg_one_branch(
+; CHECK: ret0:
+; CHECK-NEXT: ret void
+
+; CHECK: ret1:
+; CHECK-NEXT: store i32 9, i32* %val
+; CHECK-NEXT: ret void
+define void @unused_out_arg_one_branch(i1 %arg0, i32* %val) #0 {
+  br i1 %arg0, label %ret0, label %ret1
+
+ret0:
+  ret void
+
+ret1:
+  store i32 9, i32* %val
+  ret void
+}
+
+; CHECK-LABEL: define private %void_one_out_arg_v2i32_1_use @void_one_out_arg_v2i32_1_use.body(<2 x i32>* %val) #0 {
+; CHECK-NEXT: ret %void_one_out_arg_v2i32_1_use { <2 x i32> <i32 17, i32 9> }
+
+; CHECK-LABEL: define void @void_one_out_arg_v2i32_1_use(<2 x i32>*) #2 {
+; CHECK-NEXT: %2 = call %void_one_out_arg_v2i32_1_use @void_one_out_arg_v2i32_1_use.body(<2 x i32>* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_v2i32_1_use %2, 0
+; CHECK-NEXT: store <2 x i32> %3, <2 x i32>* %0, align 8
+; CHCEK-NEXT: ret void
+define void @void_one_out_arg_v2i32_1_use(<2 x i32>* %val) #0 {
+  store <2 x i32> <i32 17, i32 9>, <2 x i32>* %val
+  ret void
+}
+
+%struct = type { i32, i8, float }
+
+; CHECK-LABEL: define private %void_one_out_arg_struct_1_use @void_one_out_arg_struct_1_use.body(%struct* %out) #0 {
+; CHECK-NEXT: ret %void_one_out_arg_struct_1_use { %struct { i32 9, i8 99, float 4.000000e+00 } }
+
+; Normally this is split into element accesses which we don't handle.
+; CHECK-LABEL: define void @void_one_out_arg_struct_1_use(%struct*) #2 {
+; CHECK-NEXT: %2 = call %void_one_out_arg_struct_1_use @void_one_out_arg_struct_1_use.body(%struct* undef)
+; CHECK-NEXT: %3 = extractvalue %void_one_out_arg_struct_1_use %2, 0
+; CHECK-NEXT: store %struct %3, %struct* %0, align 4
+; CHECK-NEXT: ret void
+define void @void_one_out_arg_struct_1_use(%struct* %out) #0 {
+  store %struct { i32 9, i8 99, float 4.0 }, %struct* %out
+  ret void
+}
+
+; CHECK-LABEL: define private %i32_one_out_arg_i32_1_use @i32_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %i32_one_out_arg_i32_1_use { i32 9, i32 24 }
+
+; CHECK-LABEL: define i32 @i32_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-NEXT: %2 = call %i32_one_out_arg_i32_1_use @i32_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %i32_one_out_arg_i32_1_use %2, 1
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: %4 = extractvalue %i32_one_out_arg_i32_1_use %2, 0
+; CHECK-NEXT: ret i32 %4
+define i32 @i32_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 24, i32* %val
+  ret i32 9
+}
+
+; CHECK-LABEL: define private %unused_different_type @unused_different_type.body(i32* %arg0, float* nocapture %arg1) #0 {
+; CHECK-NEXT: ret %unused_different_type { float 4.000000e+00 }
+
+; CHECK-LABEL: define void @unused_different_type(i32*, float* nocapture) #2 {
+; CHECK-NEXT: %3 = call %unused_different_type @unused_different_type.body(i32* %0, float* undef)
+; CHECK-NEXT: %4 = extractvalue %unused_different_type %3, 0
+; CHECK-NEXT: store float %4, float* %1, align 4
+; CHECK-NEXT  ret void
+define void @unused_different_type(i32* %arg0, float* nocapture %arg1) #0 {
+  store float 4.0, float* %arg1, align 4
+  ret void
+}
+
+; CHECK-LABEL: define private %multiple_same_return_noalias @multiple_same_return_noalias.body(i32* noalias %out0, i32* noalias %out1) #0 {
+; CHECK-NEXT: ret %multiple_same_return_noalias { i32 1, i32 2 }
+
+; CHECK-LABEL: define void @multiple_same_return_noalias(
+; CHECK-NEXT: %3 = call %multiple_same_return_noalias @multiple_same_return_noalias.body(i32* undef, i32* undef)
+; CHECK-NEXT: %4 = extractvalue %multiple_same_return_noalias %3, 0
+; CHECK-NEXT: store i32 %4, i32* %0, align 4
+; CHECK-NEXT: %5 = extractvalue %multiple_same_return_noalias %3, 1
+; CHECK-NEXT: store i32 %5, i32* %1, align 4
+; CHECK-NEXT: ret void
+define void @multiple_same_return_noalias(i32* noalias %out0, i32* noalias %out1) #0 {
+  store i32 1, i32* %out0, align 4
+  store i32 2, i32* %out1, align 4
+  ret void
+}
+
+; CHECK-LABEL: define private %multiple_same_return_mayalias @multiple_same_return_mayalias.body(i32* %out0, i32* %out1) #0 {
+; CHECK-NEXT: ret %multiple_same_return_mayalias { i32 2, i32 1 }
+
+; CHECK-LABEL: define void @multiple_same_return_mayalias(i32*, i32*) #2 {
+; CHECK-NEXT: %3 = call %multiple_same_return_mayalias @multiple_same_return_mayalias.body(i32* undef, i32* undef)
+; CHECK-NEXT: %4 = extractvalue %multiple_same_return_mayalias %3, 0
+; CHECK-NEXT: store i32 %4, i32* %0, align 4
+; CHECK-NEXT: %5 = extractvalue %multiple_same_return_mayalias %3, 1
+; CHECK-NEXT: store i32 %5, i32* %1, align 4
+; CHECK-NEXT: ret void
+define void @multiple_same_return_mayalias(i32* %out0, i32* %out1) #0 {
+ store i32 1, i32* %out0, align 4
+ store i32 2, i32* %out1, align 4
+ ret void
+}
+
+; CHECK-LABEL: define private %multiple_same_return_mayalias_order @multiple_same_return_mayalias_order.body(i32* %out0, i32* %out1) #0 {
+; CHECK-NEXT: ret %multiple_same_return_mayalias_order { i32 1, i32 2 }
+
+; CHECK-LABEL: define void @multiple_same_return_mayalias_order(i32*, i32*) #2 {
+; CHECK-NEXT: %3 = call %multiple_same_return_mayalias_order @multiple_same_return_mayalias_order.body(i32* undef, i32* undef)
+; CHECK-NEXT: %4 = extractvalue %multiple_same_return_mayalias_order %3, 0
+; CHECK-NEXT: store i32 %4, i32* %0, align 4
+; CHECK-NEXT: %5 = extractvalue %multiple_same_return_mayalias_order %3, 1
+; CHECK-NEXT: store i32 %5, i32* %1, align 4
+; CHECK-NEXT: ret void
+define void @multiple_same_return_mayalias_order(i32* %out0, i32* %out1) #0 {
+ store i32 2, i32* %out1, align 4
+ store i32 1, i32* %out0, align 4
+ ret void
+}
+
+; Currently this fails to convert because the store won't be found if
+; it isn't in the same block as the return.
+; CHECK-LABEL: define i32 @store_in_entry_block(i1 %arg0, i32* %out) #0 {
+; CHECK-NOT: call
+define i32 @store_in_entry_block(i1 %arg0, i32* %out) #0 {
+entry:
+  %val0 = load i32, i32 addrspace(1)* undef
+  store i32 %val0, i32* %out
+  br i1 %arg0, label %if, label %endif
+
+if:
+  %val1 = load i32, i32 addrspace(1)* undef
+  br label %endif
+
+endif:
+  %phi = phi i32 [ 0, %entry ], [ %val1, %if ]
+  ret i32 %phi
+}
+
+; CHECK-LABEL: define private %i1_one_out_arg_i32_1_use @i1_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %i1_one_out_arg_i32_1_use { i1 true, i32 24 }
+
+; CHECK-LABEL: define i1 @i1_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK: %2 = call %i1_one_out_arg_i32_1_use @i1_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK: %3 = extractvalue %i1_one_out_arg_i32_1_use %2, 1
+; CHECK: store i32 %3, i32* %0, align 4
+; CHECK: %4 = extractvalue %i1_one_out_arg_i32_1_use %2, 0
+; CHECK: ret i1 %4
+define i1 @i1_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 24, i32* %val
+  ret i1 true
+}
+
+; Make sure we don't leave around return attributes that are
+; incompatible with struct return types.
+
+; CHECK-LABEL: define private %i1_zeroext_one_out_arg_i32_1_use @i1_zeroext_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %i1_zeroext_one_out_arg_i32_1_use { i1 true, i32 24 }
+
+; CHECK-LABEL: define zeroext i1 @i1_zeroext_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-NEXT: %2 = call %i1_zeroext_one_out_arg_i32_1_use @i1_zeroext_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %i1_zeroext_one_out_arg_i32_1_use %2, 1
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: %4 = extractvalue %i1_zeroext_one_out_arg_i32_1_use %2, 0
+; CHECK-NEXT: ret i1 %4
+define zeroext i1 @i1_zeroext_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 24, i32* %val
+  ret i1 true
+}
+
+; CHECK-LABEL: define private %i1_signext_one_out_arg_i32_1_use @i1_signext_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %i1_signext_one_out_arg_i32_1_use { i1 true, i32 24 }
+
+; CHECK-LABEL: define signext i1 @i1_signext_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-NEXT: %2 = call %i1_signext_one_out_arg_i32_1_use @i1_signext_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %i1_signext_one_out_arg_i32_1_use %2, 1
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: %4 = extractvalue %i1_signext_one_out_arg_i32_1_use %2, 0
+; CHECK-NEXT: ret i1 %4
+define signext i1 @i1_signext_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 24, i32* %val
+  ret i1 true
+}
+
+; CHECK-LABEL: define private %p1i32_noalias_one_out_arg_i32_1_use @p1i32_noalias_one_out_arg_i32_1_use.body(i32* %val) #0 {
+; CHECK-NEXT: ret %p1i32_noalias_one_out_arg_i32_1_use { i32 addrspace(1)* null, i32 24 }
+
+; CHECK-LABEL: define noalias i32 addrspace(1)* @p1i32_noalias_one_out_arg_i32_1_use(i32*) #2 {
+; CHECK-NEXT: %2 = call %p1i32_noalias_one_out_arg_i32_1_use @p1i32_noalias_one_out_arg_i32_1_use.body(i32* undef)
+; CHECK-NEXT: %3 = extractvalue %p1i32_noalias_one_out_arg_i32_1_use %2, 1
+; CHECK-NEXT: store i32 %3, i32* %0, align 4
+; CHECK-NEXT: %4 = extractvalue %p1i32_noalias_one_out_arg_i32_1_use %2, 0
+; CHECK-NEXT: ret i32 addrspace(1)* %4
+define noalias i32 addrspace(1)* @p1i32_noalias_one_out_arg_i32_1_use(i32* %val) #0 {
+  store i32 24, i32* %val
+  ret i32 addrspace(1)* null
+}
+
+; CHECK-LABEL: define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %val) #0 {
+; CHECK-NEXT: store i32 0, i32 addrspace(1)* %val
+; CHECK-NEXT: ret void
+define void @void_one_out_non_private_arg_i32_1_use(i32 addrspace(1)* %val) #0 {
+  store i32 0, i32 addrspace(1)* %val
+  ret void
+}
+
+; CHECK-LABEL: define private %func_ptr_type @func_ptr_type.body(void ()** %out) #0 {
+; CHECK-LABEL: define void @func_ptr_type(void ()**) #2 {
+; CHECK: %2 = call %func_ptr_type @func_ptr_type.body(void ()** undef)
+define void @func_ptr_type(void()** %out) #0 {
+  %func = load void()*, void()** undef
+  store void()* %func, void()** %out
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_func_ptr_type @bitcast_func_ptr_type.body(void ()** %out) #0 {
+; CHECK-LABEL: define void @bitcast_func_ptr_type(void ()**) #2 {
+define void @bitcast_func_ptr_type(void()** %out) #0 {
+  %func = load i32()*, i32()** undef
+  %cast = bitcast void()** %out to i32()**
+  store i32()* %func, i32()** %cast
+  ret void
+}
+
+; CHECK-LABEL: define private %out_arg_small_array @out_arg_small_array.body([4 x i32]* %val) #0 {
+; CHECK-NEXT: ret %out_arg_small_array { [4 x i32] [i32 0, i32 1, i32 2, i32 3] }
+
+; CHECK-LABEL: define void @out_arg_small_array([4 x i32]*) #2 {
+define void @out_arg_small_array([4 x i32]* %val) #0 {
+  store [4 x i32] [i32 0, i32 1, i32 2, i32 3], [4 x i32]* %val
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @out_arg_large_array([17 x i32]* %val) #0 {
+; CHECK-NEXT: store [17 x i32] zeroinitializer, [17 x i32]* %val
+; CHECK-NEXT: ret void
+define void @out_arg_large_array([17 x i32]* %val) #0 {
+  store [17 x i32] zeroinitializer, [17 x i32]* %val
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define <16 x i32> @num_regs_return_limit(i32* %out, i32 %val) #0 {
+define <16 x i32> @num_regs_return_limit(i32* %out, i32 %val) #0 {
+  %load = load volatile <16 x i32>, <16 x i32> addrspace(1)* undef
+  store i32 %val, i32* %out
+  ret <16 x i32> %load
+}
+
+; CHECK-LABEL: define private %num_regs_reach_limit @num_regs_reach_limit.body(i32* %out, i32 %val) #0 {
+; CHECK: define [15 x i32] @num_regs_reach_limit(i32*, i32) #2 {
+; CHECK-NEXT: call %num_regs_reach_limit @num_regs_reach_limit.body(i32* undef, i32 %1)
+define [15 x i32] @num_regs_reach_limit(i32* %out, i32 %val) #0 {
+  %load = load volatile [15 x i32], [15 x i32] addrspace(1)* undef
+  store i32 %val, i32* %out
+  ret [15 x i32] %load
+}
+
+; CHECK-LABEL: define private %num_regs_reach_limit_leftover @num_regs_reach_limit_leftover.body(i32* %out0, i32* %out1, i32 %val0) #0 {
+; CHECK-NEXT: %load0 = load volatile [15 x i32], [15 x i32] addrspace(1)* undef
+; CHECK-NEXT: %load1 = load volatile i32, i32 addrspace(1)* undef
+; CHECK-NEXT: %1 = insertvalue %num_regs_reach_limit_leftover undef, [15 x i32] %load0, 0
+; CHECK-NEXT: %2 = insertvalue %num_regs_reach_limit_leftover %1, i32 %load1, 1
+; CHECK-NEXT: %3 = insertvalue %num_regs_reach_limit_leftover %2, i32 %val0, 2
+; CHECK-NEXT: ret %num_regs_reach_limit_leftover %3
+
+; CHECK-LABEL: define [15 x i32] @num_regs_reach_limit_leftover(i32*, i32*, i32) #2 {
+; CHECK-NEXT: %4 = call %num_regs_reach_limit_leftover @num_regs_reach_limit_leftover.body(i32* undef, i32* undef, i32 %2)
+; CHECK-NEXT: %5 = extractvalue %num_regs_reach_limit_leftover %4, 1
+; CHECK-NEXT: store i32 %5, i32* %0, align 4
+; CHECK-NEXT: %6 = extractvalue %num_regs_reach_limit_leftover %4, 2
+; CHECK-NEXT: store i32 %6, i32* %1, align 4
+; CHECK-NEXT: %7 = extractvalue %num_regs_reach_limit_leftover %4, 0
+; CHECK-NEXT: ret [15 x i32] %7
+define [15 x i32] @num_regs_reach_limit_leftover(i32* %out0, i32* %out1, i32 %val0) #0 {
+  %load0 = load volatile [15 x i32], [15 x i32] addrspace(1)* undef
+  %load1 = load volatile i32, i32 addrspace(1)* undef
+  store i32 %val0, i32* %out0
+  store i32 %load1, i32* %out1
+  ret [15 x i32] %load0
+}
+
+; CHECK-LABEL: define private %preserve_debug_info @preserve_debug_info.body(i32 %arg0, i32* %val) #0 {
+; CHECK-NEXT: call void @may.clobber(), !dbg !5
+; CHECK-NEXT: %1 = insertvalue %preserve_debug_info undef, i32 %arg0, 0, !dbg !11
+; CHECK-NEXT: ret %preserve_debug_info %1, !dbg !11
+
+; CHECK-LABEL: define void @preserve_debug_info(i32, i32*) #2 !dbg !6 {
+; CHECK-NEXT: %3 = call %preserve_debug_info @preserve_debug_info.body(i32 %0, i32* undef){{$}}
+; CHECK-NEXT: %4 = extractvalue %preserve_debug_info %3, 0{{$}}
+; CHECK-NEXT: store i32 %4, i32* %1, align 4{{$}}
+; CHECK-NEXT: ret void
+define void @preserve_debug_info(i32 %arg0, i32* %val) #0 !dbg !5 {
+  call void @may.clobber(), !dbg !10
+  store i32 %arg0, i32* %val, !dbg !11
+  ret void, !dbg !12
+}
+
+define void @preserve_metadata(i32 %arg0, i32* %val) #0 !kernel_arg_access_qual !13 {
+  call void @may.clobber()
+  store i32 %arg0, i32* %val
+  ret void
+}
+
+; Clang emits this pattern for 3-vectors for some reason.
+; CHECK-LABEL: define private %bitcast_pointer_v4i32_v3i32 @bitcast_pointer_v4i32_v3i32.body(<3 x i32>* %out) #0 {
+; CHECK-NEXT: %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+; CHECK-NEXT: %bitcast = bitcast <3 x i32>* %out to <4 x i32>*
+; CHECK-NEXT: %1 = shufflevector <4 x i32> %load, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_v4i32_v3i32 undef, <3 x i32> %1, 0
+; CHECK-NEXT: ret %bitcast_pointer_v4i32_v3i32 %2
+
+; CHECK-LABEL: define void @bitcast_pointer_v4i32_v3i32(<3 x i32>*) #2 {
+; CHECK-NEXT: %2 = call %bitcast_pointer_v4i32_v3i32 @bitcast_pointer_v4i32_v3i32.body(<3 x i32>* undef)
+; CHECK-NEXT: %3 = extractvalue %bitcast_pointer_v4i32_v3i32 %2, 0
+; CHECK-NEXT: store <3 x i32> %3, <3 x i32>* %0, align 16
+; CHECK-NEXT: ret void
+define void @bitcast_pointer_v4i32_v3i32(<3 x i32>* %out) #0 {
+  %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+  %bitcast = bitcast <3 x i32>* %out to <4 x i32>*
+  store <4 x i32> %load, <4 x i32>* %bitcast
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_pointer_v4i32_v3f32 @bitcast_pointer_v4i32_v3f32.body(<3 x float>* %out) #0 {
+; CHECK-NEXT: %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+; CHECK-NEXT: %bitcast = bitcast <3 x float>* %out to <4 x i32>*
+; CHECK-NEXT: %1 = shufflevector <4 x i32> %load, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: %2 = bitcast <3 x i32> %1 to <3 x float>
+; CHECK-NEXT: %3 = insertvalue %bitcast_pointer_v4i32_v3f32 undef, <3 x float> %2, 0
+; CHECK-NEXT: ret %bitcast_pointer_v4i32_v3f32 %3
+define void @bitcast_pointer_v4i32_v3f32(<3 x float>* %out) #0 {
+  %load = load volatile <4 x i32>, <4 x i32> addrspace(1)* undef
+  %bitcast = bitcast <3 x float>* %out to <4 x i32>*
+  store <4 x i32> %load, <4 x i32>* %bitcast
+  ret void
+}
+
+
+; Try different element and bitwidths which could produce broken
+; casts.
+
+; CHECK-LABEL: define private %bitcast_pointer_i32_f32 @bitcast_pointer_i32_f32.body(float* %out) #0 {
+; CHECK-NEXT: %load = load volatile i32, i32 addrspace(1)* undef
+; CHECK-NEXT: %bitcast = bitcast float* %out to i32*
+; CHECK-NEXT: %1 = bitcast i32 %load to float
+; CHECK-NEXT: %2 = insertvalue %bitcast_pointer_i32_f32 undef, float %1, 0
+; CHECK-NEXT: ret %bitcast_pointer_i32_f32 %2
+
+; CHECK-LABEL: define void @bitcast_pointer_i32_f32(float*) #2 {
+; CHECK-NEXT: %2 = call %bitcast_pointer_i32_f32 @bitcast_pointer_i32_f32.body(float* undef)
+; CHECK-NEXT: %3 = extractvalue %bitcast_pointer_i32_f32 %2, 0
+; CHECK-NEXT: store float %3, float* %0, align 4
+define void @bitcast_pointer_i32_f32(float* %out) #0 {
+  %load = load volatile i32, i32 addrspace(1)* undef
+  %bitcast = bitcast float* %out to i32*
+  store i32 %load, i32* %bitcast
+  ret void
+}
+
+; CHECK-LABEL: define void @bitcast_pointer_i32_f16(half* %out) #0 {
+; CHECK-NOT: call
+define void @bitcast_pointer_i32_f16(half* %out) #0 {
+  %load = load volatile i32, i32 addrspace(1)* undef
+  %bitcast = bitcast half* %out to i32*
+  store i32 %load, i32* %bitcast
+  ret void
+}
+
+; CHECK-LABEL: define void @bitcast_pointer_f16_i32(i32* %out) #0 {
+; CHECK-NOT: call
+define void @bitcast_pointer_f16_i32(i32* %out) #0 {
+  %load = load volatile half, half addrspace(1)* undef
+  %bitcast = bitcast i32* %out to half*
+  store half %load, half* %bitcast
+  ret void
+}
+
+%struct.i128 = type { i128 }
+%struct.v2f32 = type { <2 x float> }
+%struct.v3f32 = type { <3 x float> }
+%struct.v3f32.f32 = type { <3 x float>, float }
+%struct.v4f32 = type { <4 x float> }
+
+; CHECK-LABEL: define private %bitcast_struct_v3f32_v3f32 @bitcast_struct_v3f32_v3f32.body(%struct.v3f32* %out, <3 x float> %value) #0 {
+; CHECK-NEXT: %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: %cast = bitcast %struct.v3f32* %out to <4 x float>*
+; CHECK-NEXT: %1 = shufflevector <4 x float> %extractVec, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: %2 = insertvalue %struct.v3f32 undef, <3 x float> %1, 0
+; CHECK-NEXT: %3 = insertvalue %bitcast_struct_v3f32_v3f32 undef, %struct.v3f32 %2, 0
+; CHECK-NEXT: ret %bitcast_struct_v3f32_v3f32 %3
+
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v3f32(%struct.v3f32*, <3 x float>) #2 {
+; CHECK-NEXT: %3 = call %bitcast_struct_v3f32_v3f32 @bitcast_struct_v3f32_v3f32.body(%struct.v3f32* undef, <3 x float> %1)
+; CHECK-NEXT: %4 = extractvalue %bitcast_struct_v3f32_v3f32 %3, 0
+; CHECK-NEXT: store %struct.v3f32 %4, %struct.v3f32* %0, align 16
+; CHECK-NEXT: ret void
+define void @bitcast_struct_v3f32_v3f32(%struct.v3f32* %out, <3 x float> %value) #0 {
+  %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %cast = bitcast %struct.v3f32* %out to <4 x float>*
+  store <4 x float> %extractVec, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_struct_v3f32_v3i32 @bitcast_struct_v3f32_v3i32.body(%struct.v3f32* %out, <3 x i32> %value) #0 {
+; CHECK-NEXT: %extractVec = shufflevector <3 x i32> %value, <3 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: %cast = bitcast %struct.v3f32* %out to <4 x i32>*
+; CHECK-NEXT: %1 = shufflevector <4 x i32> %extractVec, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: %2 = bitcast <3 x i32> %1 to <3 x float>
+; CHECK-NEXT: %3 = insertvalue %struct.v3f32 undef, <3 x float> %2, 0
+; CHECK-NEXT: %4 = insertvalue %bitcast_struct_v3f32_v3i32 undef, %struct.v3f32 %3, 0
+; CHECK-NEXT: ret %bitcast_struct_v3f32_v3i32 %4
+
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v3i32(%struct.v3f32*, <3 x i32>) #2 {
+; CHECK-NEXT: %3 = call %bitcast_struct_v3f32_v3i32 @bitcast_struct_v3f32_v3i32.body(%struct.v3f32* undef, <3 x i32> %1)
+; CHECK-NEXT: %4 = extractvalue %bitcast_struct_v3f32_v3i32 %3, 0
+; CHECK-NEXT: store %struct.v3f32 %4, %struct.v3f32* %0, align 16
+define void @bitcast_struct_v3f32_v3i32(%struct.v3f32* %out, <3 x i32> %value) #0 {
+  %extractVec = shufflevector <3 x i32> %value, <3 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %cast = bitcast %struct.v3f32* %out to <4 x i32>*
+  store <4 x i32> %extractVec, <4 x i32>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_struct_v4f32_v4f32 @bitcast_struct_v4f32_v4f32.body(%struct.v4f32* %out, <4 x float> %value) #0 {
+; CHECK-NEXT: %cast = bitcast %struct.v4f32* %out to <4 x float>*
+; CHECK-NEXT: %1 = insertvalue %struct.v4f32 undef, <4 x float> %value, 0
+; CHECK-NEXT: %2 = insertvalue %bitcast_struct_v4f32_v4f32 undef, %struct.v4f32 %1, 0
+; CHECK-NEXT: ret %bitcast_struct_v4f32_v4f32 %2
+
+; CHECK-LABEL: define void @bitcast_struct_v4f32_v4f32(%struct.v4f32*, <4 x float>) #2 {
+; CHECK-NEXT: %3 = call %bitcast_struct_v4f32_v4f32 @bitcast_struct_v4f32_v4f32.body(%struct.v4f32* undef, <4 x float> %1)
+define void @bitcast_struct_v4f32_v4f32(%struct.v4f32* %out, <4 x float> %value) #0 {
+  %cast = bitcast %struct.v4f32* %out to <4 x float>*
+  store <4 x float> %value, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_struct_v3f32_v4i32 @bitcast_struct_v3f32_v4i32.body(%struct.v3f32* %out, <4 x i32> %value) #0 {
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v4i32(%struct.v3f32*, <4 x i32>) #2 {
+define void @bitcast_struct_v3f32_v4i32(%struct.v3f32* %out, <4 x i32> %value) #0 {
+  %cast = bitcast %struct.v3f32* %out to <4 x i32>*
+  store <4 x i32> %value, <4 x i32>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_struct_v4f32_v3f32 @bitcast_struct_v4f32_v3f32.body(%struct.v4f32* %out, <3 x float> %value) #0 {
+; CHECK-LABEL: define void @bitcast_struct_v4f32_v3f32(%struct.v4f32*, <3 x float>) #2 {
+define void @bitcast_struct_v4f32_v3f32(%struct.v4f32* %out, <3 x float> %value) #0 {
+  %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %cast = bitcast %struct.v4f32* %out to <4 x float>*
+  store <4 x float> %extractVec, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @bitcast_struct_v3f32_v2f32(%struct.v3f32* %out, <2 x float> %value) #0 {
+; CHECK-NOT: call
+define void @bitcast_struct_v3f32_v2f32(%struct.v3f32* %out, <2 x float> %value) #0 {
+  %cast = bitcast %struct.v3f32* %out to <2 x float>*
+  store <2 x float> %value, <2 x float>* %cast, align 8
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @bitcast_struct_v3f32_f32_v3f32(%struct.v3f32.f32* %out, <3 x float> %value) #0 {
+; CHECK-NOT: call
+define void @bitcast_struct_v3f32_f32_v3f32(%struct.v3f32.f32* %out, <3 x float> %value) #0 {
+  %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %cast = bitcast %struct.v3f32.f32* %out to <4 x float>*
+  store <4 x float> %extractVec, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-NOT: define
+; CHECK-LABEL: define void @bitcast_struct_v3f32_f32_v4f32(%struct.v3f32.f32* %out, <4 x float> %value) #0 {
+; CHECK-NOT: call
+define void @bitcast_struct_v3f32_f32_v4f32(%struct.v3f32.f32* %out, <4 x float> %value) #0 {
+  %cast = bitcast %struct.v3f32.f32* %out to <4 x float>*
+  store <4 x float> %value, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define private %bitcast_struct_i128_v4f32 @bitcast_struct_i128_v4f32.body(%struct.i128* %out, <4 x float> %value) #0 {
+; CHECK-NEXT: %cast = bitcast %struct.i128* %out to <4 x float>*
+; CHECK-NEXT: %1 = bitcast <4 x float> %value to i128
+; CHECK-NEXT: %2 = insertvalue %struct.i128 undef, i128 %1, 0
+; CHECK-NEXT: %3 = insertvalue %bitcast_struct_i128_v4f32 undef, %struct.i128 %2, 0
+; CHECK-NEXT: ret %bitcast_struct_i128_v4f32 %3
+define void @bitcast_struct_i128_v4f32(%struct.i128* %out, <4 x float> %value) #0 {
+  %cast = bitcast %struct.i128* %out to <4 x float>*
+  store <4 x float> %value, <4 x float>* %cast, align 16
+  ret void
+}
+
+; CHECK-LABEL: define void @bitcast_struct_i128_v4f32(%struct.i128*, <4 x float>) #2 {
+; CHECK-NEXT: %3 = call %bitcast_struct_i128_v4f32 @bitcast_struct_i128_v4f32.body(%struct.i128* undef, <4 x float> %1)
+define void @bitcast_array_v4i32_v4f32([4 x i32]* %out, [4 x float] %value) #0 {
+  %cast = bitcast [4 x i32]* %out to [4 x float]*
+  store [4 x float] %value, [4 x float]* %cast, align 4
+  ret void
+}
+
+; CHECK-LABEL: define private %multi_return_bitcast_struct_v3f32_v3f32 @multi_return_bitcast_struct_v3f32_v3f32.body(i1 %cond, %struct.v3f32* %out, <3 x float> %value) #0 {
+; CHECK: ret0:
+; CHECK: %cast0 = bitcast %struct.v3f32* %out to <4 x float>*
+; CHECK: %0 = shufflevector <4 x float> %extractVec, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK: %1 = insertvalue %struct.v3f32 undef, <3 x float> %0, 0
+; CHECK: %2 = insertvalue %multi_return_bitcast_struct_v3f32_v3f32 undef, %struct.v3f32 %1, 0
+; CHECK: ret %multi_return_bitcast_struct_v3f32_v3f32 %2
+
+; CHECK: ret1:
+; CHECK: %4 = insertvalue %struct.v3f32 undef, <3 x float> %3, 0
+; CHECK: %5 = insertvalue %multi_return_bitcast_struct_v3f32_v3f32 undef, %struct.v3f32 %4, 0
+; CHECK: ret %multi_return_bitcast_struct_v3f32_v3f32 %5
+define void @multi_return_bitcast_struct_v3f32_v3f32(i1 %cond, %struct.v3f32* %out, <3 x float> %value) #0 {
+entry:
+  br i1 %cond, label %ret0, label %ret1
+
+ret0:
+  %extractVec = shufflevector <3 x float> %value, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %cast0 = bitcast %struct.v3f32* %out to <4 x float>*
+  store <4 x float> %extractVec, <4 x float>* %cast0, align 16
+  ret void
+
+ret1:
+  %cast1 = bitcast %struct.v3f32* %out to <4 x float>*
+  %load = load <4 x float>, <4 x float> addrspace(1)* undef
+  store <4 x float> %load, <4 x float>* %cast1, align 16
+  ret void
+}
+
+; CHECK-LABEL: define void @bitcast_v3f32_struct_v3f32(<3 x float>* %out, %struct.v3f32 %value) #0 {
+; CHECK-NOT: call
+define void @bitcast_v3f32_struct_v3f32(<3 x float>* %out, %struct.v3f32 %value) #0 {
+  %cast = bitcast <3 x float>* %out to %struct.v3f32*
+  store %struct.v3f32 %value, %struct.v3f32* %cast, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind noinline optnone }
+attributes #2 = { alwaysinline nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "code-object-metadata-kernel-debug-props.cl", directory: "/some/random/directory")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, variables: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null, !8}
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 2, column: 3, scope: !5)
+!11 = !DILocation(line: 2, column: 8, scope: !5)
+!12 = !DILocation(line: 3, column: 3, scope: !5)
+!13 = !{!"none"}
diff --git a/test/CodeGen/AMDGPU/rotl.i64.ll b/test/CodeGen/AMDGPU/rotl.i64.ll
index fa29d789cebee..1134f191c888b 100644
--- a/test/CodeGen/AMDGPU/rotl.i64.ll
+++ b/test/CodeGen/AMDGPU/rotl.i64.ll
@@ -20,7 +20,7 @@ entry:
 ; BOTH-LABEL: {{^}}v_rotl_i64:
 ; SI-DAG: v_lshl_b64
 ; VI-DAG: v_lshlrev_b64
-; BOTH-DAG: v_sub_i32
+; BOTH-DAG: v_sub_{{[iu]}}32
 ; SI: v_lshr_b64
 ; VI: v_lshrrev_b64
 ; BOTH: v_or_b32
diff --git a/test/CodeGen/AMDGPU/rotr.i64.ll b/test/CodeGen/AMDGPU/rotr.i64.ll
index af58b404ca6c6..c55af2376ff0f 100644
--- a/test/CodeGen/AMDGPU/rotr.i64.ll
+++ b/test/CodeGen/AMDGPU/rotr.i64.ll
@@ -17,7 +17,7 @@ entry:
 }
 
 ; BOTH-LABEL: {{^}}v_rotr_i64:
-; BOTH-DAG: v_sub_i32
+; BOTH-DAG: v_sub_{{[iu]}}32
 ; SI-DAG: v_lshr_b64
 ; SI-DAG: v_lshl_b64
 ; VI-DAG: v_lshrrev_b64
diff --git a/test/CodeGen/AMDGPU/s_addk_i32.ll b/test/CodeGen/AMDGPU/s_addk_i32.ll
index deef24cea3775..ac7a33aff5260 100644
--- a/test/CodeGen/AMDGPU/s_addk_i32.ll
+++ b/test/CodeGen/AMDGPU/s_addk_i32.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 ; SI-LABEL: {{^}}s_addk_i32_k0:
diff --git a/test/CodeGen/AMDGPU/s_mulk_i32.ll b/test/CodeGen/AMDGPU/s_mulk_i32.ll
index f6ed5408ba453..78f5e8afef4cc 100644
--- a/test/CodeGen/AMDGPU/s_mulk_i32.ll
+++ b/test/CodeGen/AMDGPU/s_mulk_i32.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 ; SI-LABEL: {{^}}s_mulk_i32_k0:
diff --git a/test/CodeGen/AMDGPU/sad.ll b/test/CodeGen/AMDGPU/sad.ll
index ee56e9053fd3f..5d981a225d741 100644
--- a/test/CodeGen/AMDGPU/sad.ll
+++ b/test/CodeGen/AMDGPU/sad.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}v_sad_u32_pat1:
 ; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/saddo.ll b/test/CodeGen/AMDGPU/saddo.ll
index 09e87d524419e..0b52821f72cc9 100644
--- a/test/CodeGen/AMDGPU/saddo.ll
+++ b/test/CodeGen/AMDGPU/saddo.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs< %s
 
 declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
@@ -49,8 +50,11 @@ define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_saddo_i64:
-; SI: v_add_i32
-; SI: v_addc_u32
+; SICIVI: v_add_{{[iu]}}32_e32 v{{[0-9]+}}, vcc
+; SICIVI: v_addc_u32_e32 v{{[0-9]+}}, vcc
+
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc
 define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
diff --git a/test/CodeGen/AMDGPU/salu-to-valu.ll b/test/CodeGen/AMDGPU/salu-to-valu.ll
index d5b2fa0b67540..a826661442e10 100644
--- a/test/CodeGen/AMDGPU/salu-to-valu.ll
+++ b/test/CodeGen/AMDGPU/salu-to-valu.ll
@@ -170,10 +170,10 @@ entry:
 ; CI.
 
 ; GCN-LABEL: {{^}}smrd_valu_ci_offset_x8:
-; GCN-NOHSA: s_mov_b32 [[OFFSET1:s[0-9]+]], 0x9a50{{$}}
-; GCN-NOHSA-NOT: v_add
 ; GCN-NOHSA: s_mov_b32 [[OFFSET0:s[0-9]+]], 0x9a40{{$}}
 ; GCN-NOHSA-NOT: v_add
+; GCN-NOHSA: s_mov_b32 [[OFFSET1:s[0-9]+]], 0x9a50{{$}}
+; GCN-NOHSA-NOT: v_add
 ; GCN-NOHSA: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET1]] addr64{{$}}
 ; GCN-NOHSA: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET0]] addr64{{$}}
 
diff --git a/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
new file mode 100644
index 0000000000000..a89011a0cce01
--- /dev/null
+++ b/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
@@ -0,0 +1,333 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa-opencl -verify-machineinstrs -run-pass=machine-scheduler -o - %s | FileCheck %s
+
+--- |
+  %struct.widget.0 = type { float, i32, i32 }
+  %struct.baz = type { <4 x float>, <4 x float>, <2 x float>, i32, i32 }
+  %struct.snork = type { float, float, float, i32, float, float, float, float, %struct.spam }
+  %struct.spam = type { %struct.zot, [16 x i8] }
+  %struct.zot = type { float, float, float, float, <4 x float> }
+  %struct.wombat = type { <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, <4 x float>, [2 x i16], [2 x i16] }
+  %struct.wombat.1 = type { [4 x i32], [4 x i32], [4 x i32], [4 x i32], i32, i32, i32, i32 }
+
+  @sched_dbg_value_crash.tmp6 = internal unnamed_addr addrspace(3) global [256 x [16 x i8]] undef, align 16
+
+  define amdgpu_kernel void @sched_dbg_value_crash(i8 addrspace(1)* nocapture readonly %arg, i32 addrspace(1)* nocapture readonly %arg1, %struct.widget.0 addrspace(1)* nocapture readonly %arg2, %struct.baz addrspace(1)* nocapture readonly %arg3, %struct.snork addrspace(1)* nocapture %arg4) local_unnamed_addr #2 {
+  bb:
+    %0 = getelementptr i32, i32 addrspace(1)* %arg1, i64 0, !amdgpu.uniform !3, !amdgpu.noclobber !3
+    %tmp5 = alloca %struct.wombat, align 16
+    %1 = call noalias nonnull dereferenceable(64) i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+    %2 = bitcast i8 addrspace(2)* %1 to i32 addrspace(2)*
+    %3 = getelementptr inbounds i32, i32 addrspace(2)* %2, i64 1
+    %4 = bitcast i32 addrspace(2)* %3 to <2 x i32> addrspace(2)*, !amdgpu.uniform !3, !amdgpu.noclobber !3
+    %5 = load <2 x i32>, <2 x i32> addrspace(2)* %4, align 4, !invariant.load !3
+    %6 = extractelement <2 x i32> %5, i32 0
+    %7 = extractelement <2 x i32> %5, i32 1
+    %8 = lshr i32 %6, 16
+    %9 = call i32 @llvm.amdgcn.workitem.id.x(), !range !4
+    %10 = call i32 @llvm.amdgcn.workitem.id.y(), !range !4
+    %11 = call i32 @llvm.amdgcn.workitem.id.z(), !range !4
+    %12 = mul nuw nsw i32 %8, %7
+    %13 = mul i32 %12, %9
+    %14 = mul nuw nsw i32 %10, %7
+    %15 = add i32 %13, %14
+    %16 = add i32 %15, %11
+    %17 = getelementptr inbounds [256 x [16 x i8]], [256 x [16 x i8]] addrspace(3)* @sched_dbg_value_crash.tmp6, i32 0, i32 %16
+    %tmp7 = load i64, i64 addrspace(2)* null, align 536870912
+    %tmp8 = tail call i32 @llvm.amdgcn.workitem.id.x() #3, !range !4
+    %tmp9 = zext i32 %tmp8 to i64
+    %tmp10 = add i64 %tmp7, %tmp9
+    %tmp11 = shl i64 %tmp10, 32
+    %tmp12 = ashr exact i64 %tmp11, 32
+    %tmp13 = getelementptr inbounds %struct.widget.0, %struct.widget.0 addrspace(1)* %arg2, i64 %tmp12, i32 1
+    %tmp14 = load i32, i32 addrspace(1)* %tmp13, align 4
+    %tmp15 = getelementptr inbounds %struct.baz, %struct.baz addrspace(1)* %arg3, i64 %tmp12, i32 1
+    %tmp16 = load <4 x float>, <4 x float> addrspace(1)* %tmp15, align 16
+    %tmp17 = sext i32 %tmp14 to i64
+    %tmp18 = load i32, i32 addrspace(1)* %0, align 4
+    %tmp19 = zext i32 %tmp18 to i64
+    %tmp20 = shl nuw nsw i64 %tmp19, 2
+    %tmp21 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 %tmp20
+    %tmp22 = bitcast i8 addrspace(1)* %tmp21 to %struct.wombat.1 addrspace(1)*
+    %tmp23 = bitcast %struct.wombat* %tmp5 to i8*
+    call void @llvm.lifetime.start.p0i8(i64 144, i8* nonnull %tmp23) #3
+    %tmp24 = getelementptr inbounds %struct.wombat, %struct.wombat* %tmp5, i32 0, i32 6
+    %tmp25 = getelementptr i32, i32 addrspace(1)* %arg1, i64 3, !amdgpu.uniform !3, !amdgpu.noclobber !3
+    %tmp26 = load i32, i32 addrspace(1)* %tmp25, align 4
+    %tmp27 = zext i32 %tmp26 to i64
+    %tmp28 = shl nuw nsw i64 %tmp27, 2
+    %tmp29 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 %tmp28
+    %tmp30 = bitcast i8 addrspace(1)* %tmp29 to <2 x float> addrspace(1)*
+    %tmp31 = getelementptr inbounds %struct.wombat.1, %struct.wombat.1 addrspace(1)* %tmp22, i64 %tmp17, i32 2, i64 0
+    %18 = bitcast i32 addrspace(1)* %tmp31 to <3 x i32> addrspace(1)*
+    %19 = load <3 x i32>, <3 x i32> addrspace(1)* %18, align 4
+    %tmp325 = extractelement <3 x i32> %19, i32 0
+    %tmp386 = extractelement <3 x i32> %19, i32 1
+    %tmp447 = extractelement <3 x i32> %19, i32 2
+    %tmp33 = sext i32 %tmp325 to i64
+    %tmp34 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %tmp30, i64 %tmp33
+    %tmp35 = load <2 x float>, <2 x float> addrspace(1)* %tmp34, align 8
+    %tmp36 = extractelement <2 x float> %tmp35, i32 1
+    %tmp39 = sext i32 %tmp386 to i64
+    %tmp40 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %tmp30, i64 %tmp39
+    %tmp41 = load <2 x float>, <2 x float> addrspace(1)* %tmp40, align 8
+    %tmp42 = extractelement <2 x float> %tmp41, i32 1
+    %tmp45 = sext i32 %tmp447 to i64
+    %tmp46 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %tmp30, i64 %tmp45
+    %tmp47 = load <2 x float>, <2 x float> addrspace(1)* %tmp46, align 8
+    %tmp48 = extractelement <2 x float> %tmp47, i32 1
+    %tmp49 = getelementptr i32, i32 addrspace(1)* %arg1, i64 1, !amdgpu.uniform !3, !amdgpu.noclobber !3
+    %tmp50 = load i32, i32 addrspace(1)* %tmp49, align 4
+    %tmp51 = zext i32 %tmp50 to i64
+    %tmp52 = shl nuw nsw i64 %tmp51, 2
+    %tmp53 = getelementptr inbounds i8, i8 addrspace(1)* %arg, i64 %tmp52
+    %tmp54 = bitcast i8 addrspace(1)* %tmp53 to <4 x float> addrspace(1)*
+    %tmp55 = getelementptr inbounds %struct.wombat.1, %struct.wombat.1 addrspace(1)* %tmp22, i64 %tmp17, i32 0, i64 0
+    %20 = bitcast i32 addrspace(1)* %tmp55 to <2 x i32> addrspace(1)*
+    %21 = load <2 x i32>, <2 x i32> addrspace(1)* %20, align 4
+    %tmp568 = extractelement <2 x i32> %21, i32 0
+    %tmp639 = extractelement <2 x i32> %21, i32 1
+    %tmp57 = sext i32 %tmp568 to i64
+    %tmp58 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %tmp54, i64 %tmp57
+    %tmp59 = load <4 x float>, <4 x float> addrspace(1)* %tmp58, align 16
+    %tmp60 = extractelement <4 x float> %tmp59, i32 0
+    %tmp61 = extractelement <4 x float> %tmp59, i32 1
+    %tmp64 = sext i32 %tmp639 to i64
+    %tmp65 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %tmp54, i64 %tmp64
+    %tmp66 = load <4 x float>, <4 x float> addrspace(1)* %tmp65, align 16
+    %tmp67 = extractelement <4 x float> %tmp16, i64 0
+    %tmp69 = fsub fast float -0.000000e+00, %tmp67
+    %tmp70 = fmul float %tmp67, 0.000000e+00
+    %tmp = fmul fast float %tmp67, undef
+    %tmp71 = fsub fast float %tmp, %tmp70
+    %tmp73 = fadd fast float %tmp, undef
+    %tmp74 = insertelement <4 x float> <float undef, float undef, float undef, float 0.000000e+00>, float %tmp69, i32 0
+    %tmp75 = insertelement <4 x float> %tmp74, float %tmp71, i32 1
+    %tmp76 = insertelement <4 x float> %tmp75, float %tmp73, i32 2
+    store <4 x float> %tmp76, <4 x float>* %tmp24, align 16
+    %tmp77 = fsub float undef, %tmp60
+    %tmp78 = fsub float undef, %tmp61
+    %tmp79 = extractelement <4 x float> %tmp66, i32 2
+    %tmp80 = extractelement <4 x float> %tmp59, i32 2
+    %tmp81 = fsub float %tmp79, %tmp80
+    %tmp82 = fmul fast float %tmp81, undef
+    %tmp83 = fmul fast float %tmp78, undef
+    %tmp84 = fadd fast float %tmp83, %tmp77
+    %tmp85 = fadd fast float %tmp84, undef
+    %tmp86 = fmul float %tmp82, %tmp82
+    %tmp87 = fdiv float 1.000000e+00, %tmp86
+    tail call void @llvm.dbg.value(metadata float %tmp87, metadata !5, metadata !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef)) #3, !dbg !8
+    %tmp88 = fmul float %tmp82, 0.000000e+00
+    %tmp89 = fsub fast float %tmp85, %tmp88
+    %tmp90 = fdiv float %tmp89, %tmp86
+    %tmp91 = fsub float 1.000000e+00, %tmp87
+    %tmp92 = fsub float %tmp91, %tmp90
+    %tmp93 = fmul float %tmp42, %tmp87
+    %tmp94 = call float @llvm.fmuladd.f32(float %tmp92, float %tmp36, float %tmp93)
+    %tmp95 = call float @llvm.fmuladd.f32(float %tmp48, float undef, float %tmp94)
+    %tmp96 = fsub float extractelement (<2 x float> fadd (<2 x float> fmul (<2 x float> undef, <2 x float> undef), <2 x float> undef), i64 1), %tmp95
+    %tmp97 = getelementptr inbounds %struct.wombat, %struct.wombat* %tmp5, i32 0, i32 8, i32 1
+    call void @func(float %tmp96, i64 0, i16* nonnull %tmp97) #3
+    %tmp984 = bitcast [16 x i8] addrspace(3)* %17 to i8 addrspace(3)*
+    %tmp99 = getelementptr inbounds %struct.snork, %struct.snork addrspace(1)* %arg4, i64 %tmp12, i32 8, i32 1, i64 0
+    call void @llvm.memcpy.p1i8.p3i8.i64(i8 addrspace(1)* %tmp99, i8 addrspace(3)* %tmp984, i64 16, i32 16, i1 false)
+    call void @llvm.lifetime.end.p0i8(i64 144, i8* nonnull %tmp23) #3
+    ret void
+  }
+
+  declare void @func(float, i64, i16*)
+  declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #0
+  declare float @llvm.fmuladd.f32(float, float, float) #1
+  declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #0
+  declare <2 x float> @llvm.fmuladd.v2f32(<2 x float>, <2 x float>, <2 x float>) #1
+  declare i32 @llvm.amdgcn.workitem.id.x() #1
+  declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+  declare i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #1
+  declare i32 @llvm.amdgcn.workitem.id.y() #1
+  declare i32 @llvm.amdgcn.workitem.id.z() #1
+  declare void @llvm.memcpy.p1i8.p0i8.i64(i8 addrspace(1)* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #0
+  declare void @llvm.memcpy.p1i8.p3i8.i64(i8 addrspace(1)* nocapture writeonly, i8 addrspace(3)* nocapture readonly, i64, i32, i1) #0
+
+  attributes #0 = { argmemonly nounwind }
+  attributes #1 = { nounwind readnone speculatable }
+  attributes #2 = { convergent nounwind "amdgpu-dispatch-ptr" "amdgpu-flat-scratch" "amdgpu-work-item-id-y" "amdgpu-work-item-id-z" "target-cpu"="gfx900" "target-features"="+fp32-denormals" }
+  attributes #3 = { nounwind }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!2}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+  !1 = !DIFile(filename: "foo.cl", directory: "/dev/null")
+  !2 = !{i32 2, !"Debug Info Version", i32 3}
+  !3 = !{}
+  !4 = !{i32 0, i32 256}
+  !5 = !DILocalVariable(name: "bar", scope: !6, file: !1, line: 102, type: !7)
+  !6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 81, isLocal: false, isDefinition: true, scopeLine: 86, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
+  !7 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+  !8 = !DILocation(line: 102, column: 8, scope: !6)
+
+...
+---
+
+# CHECK: name: sched_dbg_value_crash
+# CHECK: DBG_VALUE debug-use %99, debug-use %noreg, !5, !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef), debug-location !8
+
+name:            sched_dbg_value_crash
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '%0' }
+  - { reg: '%vgpr1', virtual-reg: '%1' }
+  - { reg: '%vgpr2', virtual-reg: '%2' }
+  - { reg: '%sgpr4_sgpr5', virtual-reg: '%3' }
+  - { reg: '%sgpr6_sgpr7', virtual-reg: '%4' }
+fixedStack:
+stack:
+  - { id: 0, name: tmp5, type: default, offset: 0, size: 128, alignment: 16,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: 0, di-variable: '', di-expression: '', di-location: '' }
+constants:
+body:             |
+  bb.0.bb:
+    liveins: %vgpr0, %vgpr1, %vgpr2, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4_sgpr5, %sgpr6_sgpr7, %sgpr32, %sgpr101
+
+    %4:sgpr_64 = COPY %sgpr6_sgpr7
+    %3:sgpr_64 = COPY %sgpr4_sgpr5
+    %2:vgpr_32 = COPY %vgpr2
+    %1:vgpr_32 = COPY %vgpr1
+    %0:vgpr_32 = COPY %vgpr0
+    %5:sreg_64_xexec = S_LOAD_DWORDX2_IMM %4, 0, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
+    %6:sreg_64_xexec = S_LOAD_DWORDX2_IMM %4, 8, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
+    %7:sreg_64_xexec = S_LOAD_DWORDX2_IMM %4, 16, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
+    %8:sreg_64_xexec = S_LOAD_DWORDX2_IMM %4, 24, 0
+    %9:sreg_64_xexec = S_LOAD_DWORDX2_IMM %4, 32, 0
+    %10:sreg_64_xexec = S_LOAD_DWORDX2_IMM %3, 4, 0
+    %11:sreg_32_xm0 = S_LSHR_B32 %10.sub0, 16, implicit-def dead %scc
+    %12:sreg_32_xm0 = S_MUL_I32 %11, %10.sub1
+    %13:vgpr_32 = V_MUL_LO_I32 0, %0, implicit %exec
+    %14:vgpr_32 = V_MUL_LO_I32 %1, %10.sub1, implicit %exec
+    %15:vgpr_32 = V_ADD_I32_e32 0, %13, implicit-def dead %vcc, implicit %exec
+    %16:vgpr_32 = V_ADD_I32_e32 0, %15, implicit-def dead %vcc, implicit %exec
+    %17:vgpr_32 = IMPLICIT_DEF
+    %18:sreg_64 = S_MOV_B64 0
+    %19:sreg_32_xm0_xexec = IMPLICIT_DEF
+    %20:vgpr_32 = V_ADD_I32_e32 %19, %0, implicit-def dead %vcc, implicit %exec
+    %21:vreg_64, dead %22:sreg_64 = V_MAD_I64_I32 %20, 12, %7, 0, implicit %exec
+    %23:vgpr_32 = GLOBAL_LOAD_DWORD %21, 4, 0, 0, implicit %exec
+    %24:vreg_64, dead %25:sreg_64 = V_MAD_I64_I32 %20, 48, %8, 0, implicit %exec
+    %26:vreg_128 = IMPLICIT_DEF
+    undef %27.sub0:sreg_64_xexec = S_LOAD_DWORD_IMM %6, 0, 0
+    %27.sub1:sreg_64_xexec = S_MOV_B32 0
+    %28:sreg_64 = S_LSHL_B64 %27, 2, implicit-def dead %scc
+    undef %29.sub0:sreg_64 = S_ADD_U32 %5.sub0, %28.sub0, implicit-def %scc
+    %29.sub1:sreg_64 = S_ADDC_U32 %5.sub1, %28.sub1, implicit-def dead %scc, implicit killed %scc
+    undef %30.sub0:sreg_64_xexec = S_LOAD_DWORD_IMM %6, 4, 0
+    %27.sub0:sreg_64_xexec = IMPLICIT_DEF
+    %31:sreg_64 = S_LSHL_B64 %27, 2, implicit-def dead %scc
+    %32:sreg_32_xm0 = S_ADD_U32 0, %31.sub0, implicit-def %scc
+    %33:sgpr_32 = S_ADDC_U32 %5.sub1, %31.sub1, implicit-def dead %scc, implicit killed %scc
+    %34:vgpr_32 = IMPLICIT_DEF
+    %35:vreg_64, dead %36:sreg_64 = V_MAD_I64_I32 %23, %34, 0, 0, implicit %exec
+    %37:vreg_64 = GLOBAL_LOAD_DWORDX2 %35, 32, 0, 0, implicit %exec
+    undef %38.sub1:vreg_64 = V_ASHRREV_I32_e32 31, %37.sub0, implicit %exec
+    %38.sub0:vreg_64 = COPY %37.sub0
+    %39:vreg_64 = V_LSHLREV_B64 3, %38, implicit %exec
+    undef %40.sub0:vreg_64, %41:sreg_64_xexec = V_ADD_I32_e64 0, %39.sub0, implicit %exec
+    %42:vgpr_32 = COPY %33
+    %40.sub1:vreg_64, dead %43:sreg_64_xexec = V_ADDC_U32_e64 %42, %39.sub1, %41, implicit %exec
+    %44:vreg_64 = GLOBAL_LOAD_DWORDX2 %40, 0, 0, 0, implicit %exec :: (load 8 from %ir.tmp34)
+    undef %45.sub1:vreg_64 = IMPLICIT_DEF
+    %45.sub0:vreg_64 = COPY %37.sub1
+    %46:vreg_64 = V_LSHLREV_B64 3, %45, implicit %exec
+    undef %47.sub0:vreg_64, %48:sreg_64_xexec = V_ADD_I32_e64 %32, %46.sub0, implicit %exec
+    %49:vgpr_32 = COPY %33
+    %47.sub1:vreg_64, dead %50:sreg_64_xexec = V_ADDC_U32_e64 %49, %46.sub1, %48, implicit %exec
+    %51:vreg_64 = IMPLICIT_DEF
+    undef %52.sub0:vreg_64 = GLOBAL_LOAD_DWORD %35, 40, 0, 0, implicit %exec :: (load 4 from %ir.18 + 8)
+    %52.sub1:vreg_64 = IMPLICIT_DEF
+    %53:vreg_64 = V_LSHLREV_B64 3, %52, implicit %exec
+    undef %54.sub0:vreg_64, %55:sreg_64_xexec = V_ADD_I32_e64 0, %53.sub0, implicit %exec
+    %56:vgpr_32 = COPY %33
+    %54.sub1:vreg_64, dead %57:sreg_64_xexec = V_ADDC_U32_e64 0, %53.sub1, %55, implicit %exec
+    %58:vreg_64 = IMPLICIT_DEF
+    %30.sub1:sreg_64_xexec = IMPLICIT_DEF
+    %59:sreg_64 = IMPLICIT_DEF
+    %60:sreg_32_xm0 = S_ADD_U32 %5.sub0, %59.sub0, implicit-def %scc
+    %61:sgpr_32 = S_ADDC_U32 %5.sub1, %59.sub1, implicit-def dead %scc, implicit killed %scc
+    %62:vreg_64 = GLOBAL_LOAD_DWORDX2 %35, 0, 0, 0, implicit %exec :: (load 8 from %ir.20, align 4)
+    undef %63.sub1:vreg_64 = V_ASHRREV_I32_e32 31, %62.sub0, implicit %exec
+    %63.sub0:vreg_64 = COPY %62.sub0
+    %64:vreg_64 = IMPLICIT_DEF
+    undef %65.sub0:vreg_64, %66:sreg_64_xexec = V_ADD_I32_e64 %60, %64.sub0, implicit %exec
+    %67:vgpr_32 = COPY %61
+    %65.sub1:vreg_64, dead %68:sreg_64_xexec = V_ADDC_U32_e64 %67, %64.sub1, %66, implicit %exec
+    %69:vreg_128 = GLOBAL_LOAD_DWORDX4 %65, 0, 0, 0, implicit %exec :: (load 16 from %ir.tmp58)
+    undef %70.sub1:vreg_64 = IMPLICIT_DEF
+    %70.sub0:vreg_64 = IMPLICIT_DEF
+    %71:vreg_64 = IMPLICIT_DEF
+    undef %72.sub0:vreg_64, %73:sreg_64_xexec = V_ADD_I32_e64 %60, %71.sub0, implicit %exec
+    %74:vgpr_32 = COPY %61
+    %72.sub1:vreg_64, dead %75:sreg_64_xexec = V_ADDC_U32_e64 0, %71.sub1, %73, implicit %exec
+    %76:vreg_128 = GLOBAL_LOAD_DWORDX4 %72, 0, 0, 0, implicit %exec
+    %77:vgpr_32 = IMPLICIT_DEF
+    %78:vgpr_32 = IMPLICIT_DEF
+    %79:vgpr_32 = V_MUL_F32_e32 0, %77, implicit %exec
+    %80:vgpr_32 = IMPLICIT_DEF
+    %81:vgpr_32 = IMPLICIT_DEF
+    %84:vgpr_32 = IMPLICIT_DEF
+    BUFFER_STORE_DWORD_OFFEN %84, %stack.0.tmp5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr101, 108, 0, 0, 0, implicit %exec
+    BUFFER_STORE_DWORD_OFFEN %81, %stack.0.tmp5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr101, 104, 0, 0, 0, implicit %exec
+    BUFFER_STORE_DWORD_OFFEN %80, %stack.0.tmp5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr101, 100, 0, 0, 0, implicit %exec
+    BUFFER_STORE_DWORD_OFFEN %78, %stack.0.tmp5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr101, 96, 0, 0, 0, implicit %exec
+    %85:vgpr_32 = IMPLICIT_DEF
+    %86:vgpr_32 = IMPLICIT_DEF
+    %87:vgpr_32 = IMPLICIT_DEF
+    %88:vgpr_32 = IMPLICIT_DEF
+    %90:vgpr_32 = IMPLICIT_DEF
+    %91:vgpr_32, dead %92:sreg_64 = V_DIV_SCALE_F32 %90, %90, 1065353216, implicit %exec
+    %95:vgpr_32 = V_FMA_F32 0, 0, 0, 0, 0, undef %93:vgpr_32, 0, 0, implicit %exec
+    %96:vgpr_32, %97:sreg_64 = V_DIV_SCALE_F32 1065353216, %90, 1065353216, implicit %exec
+    %98:vgpr_32 = IMPLICIT_DEF
+    %99:vgpr_32 = IMPLICIT_DEF
+    %100:vgpr_32 = IMPLICIT_DEF
+    %101:vgpr_32 = IMPLICIT_DEF
+    %102:vgpr_32 = IMPLICIT_DEF
+    %103:vgpr_32 = IMPLICIT_DEF
+    %104:vgpr_32 = IMPLICIT_DEF
+    %105:vgpr_32 = IMPLICIT_DEF
+    %106:vgpr_32, dead %107:sreg_64 = V_DIV_SCALE_F32 %90, %90, %105, implicit %exec
+    %108:vgpr_32 = V_RCP_F32_e32 0, implicit %exec
+    %109:vgpr_32 = IMPLICIT_DEF
+    %110:vgpr_32 = V_FMA_F32 0, 0, 0, 0, 0, 0, 0, 0, implicit %exec
+    %111:vgpr_32, %112:sreg_64 = V_DIV_SCALE_F32 0, 0, 0, implicit %exec
+    %113:vgpr_32 = V_MUL_F32_e32 0, %110, implicit %exec
+    %114:vgpr_32 = IMPLICIT_DEF
+    %115:vgpr_32 = IMPLICIT_DEF
+    %116:vgpr_32 = IMPLICIT_DEF
+    %vcc = IMPLICIT_DEF
+    %117:vgpr_32 = V_DIV_FMAS_F32 0, %116, 0, %110, 0, %115, 0, 0, implicit killed %vcc, implicit %exec
+    %118:vgpr_32 = V_DIV_FIXUP_F32 0, %117, 0, %90, 0, %105, 0, 0, implicit %exec
+    %119:vgpr_32 = IMPLICIT_DEF
+    %120:vgpr_32 = IMPLICIT_DEF
+    %121:vgpr_32 = IMPLICIT_DEF
+    %122:vgpr_32 = IMPLICIT_DEF
+    %123:vgpr_32 = IMPLICIT_DEF
+    %124:vgpr_32 = IMPLICIT_DEF
+    %125:vgpr_32 = IMPLICIT_DEF
+    %126:vgpr_32 = IMPLICIT_DEF
+    DBG_VALUE debug-use %103, debug-use _, !5, !DIExpression(DW_OP_constu, 1, DW_OP_swap, DW_OP_xderef), debug-location !8
+    ADJCALLSTACKUP 0, 0, implicit-def %sgpr32, implicit %sgpr32
+    %127:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @func + 4, target-flags(amdgpu-rel32-hi) @func + 4, implicit-def dead %scc
+    %sgpr4 = COPY %sgpr101
+    %vgpr0 = COPY %124
+    %vgpr1_vgpr2 = IMPLICIT_DEF
+    %vgpr3 = COPY %126
+    dead %sgpr30_sgpr31 = SI_CALL %127, @func, csr_amdgpu_highregs, implicit %sgpr0_sgpr1_sgpr2_sgpr3, implicit %sgpr4, implicit %vgpr0, implicit %vgpr1_vgpr2, implicit killed %vgpr3
+    ADJCALLSTACKDOWN 0, 0, implicit-def %sgpr32, implicit %sgpr32
+    %128:vreg_64, dead %129:sreg_64 = V_MAD_I64_I32 %20, %34, 0, 0, implicit %exec
+    S_ENDPGM
+
+...
diff --git a/test/CodeGen/AMDGPU/schedule-ilp.ll b/test/CodeGen/AMDGPU/schedule-ilp.ll
new file mode 100644
index 0000000000000..a313664cb0e5e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/schedule-ilp.ll
@@ -0,0 +1,589 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -misched=gcn-ilp -verify-machineinstrs < %s | FileCheck %s
+
+; CHECK: NumVgprs: {{[0-9][0-9][0-9]$}}
+
+define amdgpu_kernel void @load_fma_store(float addrspace(3)* nocapture readonly %arg, float addrspace(1)* nocapture %arg1) #0 {
+bb:
+  %tmp = getelementptr inbounds float, float addrspace(3)* %arg, i32 1
+  %tmp2 = load float, float addrspace(3)* %tmp, align 4
+  %tmp3 = getelementptr inbounds float, float addrspace(3)* %arg, i32 2
+  %tmp4 = load float, float addrspace(3)* %tmp3, align 4
+  %tmp5 = getelementptr inbounds float, float addrspace(3)* %arg, i32 3
+  %tmp6 = load float, float addrspace(3)* %tmp5, align 4
+  %tmp7 = tail call float @llvm.fmuladd.f32(float %tmp2, float %tmp4, float %tmp6)
+  %tmp8 = getelementptr inbounds float, float addrspace(3)* %arg, i32 5
+  %tmp9 = load float, float addrspace(3)* %tmp8, align 4
+  %tmp10 = getelementptr inbounds float, float addrspace(3)* %arg, i32 6
+  %tmp11 = load float, float addrspace(3)* %tmp10, align 4
+  %tmp12 = getelementptr inbounds float, float addrspace(3)* %arg, i32 7
+  %tmp13 = load float, float addrspace(3)* %tmp12, align 4
+  %tmp14 = tail call float @llvm.fmuladd.f32(float %tmp9, float %tmp11, float %tmp13)
+  %tmp15 = getelementptr inbounds float, float addrspace(3)* %arg, i32 9
+  %tmp16 = load float, float addrspace(3)* %tmp15, align 4
+  %tmp17 = getelementptr inbounds float, float addrspace(3)* %arg, i32 10
+  %tmp18 = load float, float addrspace(3)* %tmp17, align 4
+  %tmp19 = getelementptr inbounds float, float addrspace(3)* %arg, i32 11
+  %tmp20 = load float, float addrspace(3)* %tmp19, align 4
+  %tmp21 = tail call float @llvm.fmuladd.f32(float %tmp16, float %tmp18, float %tmp20)
+  %tmp22 = getelementptr inbounds float, float addrspace(3)* %arg, i32 13
+  %tmp23 = load float, float addrspace(3)* %tmp22, align 4
+  %tmp24 = getelementptr inbounds float, float addrspace(3)* %arg, i32 14
+  %tmp25 = load float, float addrspace(3)* %tmp24, align 4
+  %tmp26 = getelementptr inbounds float, float addrspace(3)* %arg, i32 15
+  %tmp27 = load float, float addrspace(3)* %tmp26, align 4
+  %tmp28 = tail call float @llvm.fmuladd.f32(float %tmp23, float %tmp25, float %tmp27)
+  %tmp29 = getelementptr inbounds float, float addrspace(3)* %arg, i32 17
+  %tmp30 = load float, float addrspace(3)* %tmp29, align 4
+  %tmp31 = getelementptr inbounds float, float addrspace(3)* %arg, i32 18
+  %tmp32 = load float, float addrspace(3)* %tmp31, align 4
+  %tmp33 = getelementptr inbounds float, float addrspace(3)* %arg, i32 19
+  %tmp34 = load float, float addrspace(3)* %tmp33, align 4
+  %tmp35 = tail call float @llvm.fmuladd.f32(float %tmp30, float %tmp32, float %tmp34)
+  %tmp36 = getelementptr inbounds float, float addrspace(3)* %arg, i32 21
+  %tmp37 = load float, float addrspace(3)* %tmp36, align 4
+  %tmp38 = getelementptr inbounds float, float addrspace(3)* %arg, i32 22
+  %tmp39 = load float, float addrspace(3)* %tmp38, align 4
+  %tmp40 = getelementptr inbounds float, float addrspace(3)* %arg, i32 23
+  %tmp41 = load float, float addrspace(3)* %tmp40, align 4
+  %tmp42 = tail call float @llvm.fmuladd.f32(float %tmp37, float %tmp39, float %tmp41)
+  %tmp43 = getelementptr inbounds float, float addrspace(3)* %arg, i32 25
+  %tmp44 = load float, float addrspace(3)* %tmp43, align 4
+  %tmp45 = getelementptr inbounds float, float addrspace(3)* %arg, i32 26
+  %tmp46 = load float, float addrspace(3)* %tmp45, align 4
+  %tmp47 = getelementptr inbounds float, float addrspace(3)* %arg, i32 27
+  %tmp48 = load float, float addrspace(3)* %tmp47, align 4
+  %tmp49 = tail call float @llvm.fmuladd.f32(float %tmp44, float %tmp46, float %tmp48)
+  %tmp50 = getelementptr inbounds float, float addrspace(3)* %arg, i32 29
+  %tmp51 = load float, float addrspace(3)* %tmp50, align 4
+  %tmp52 = getelementptr inbounds float, float addrspace(3)* %arg, i32 30
+  %tmp53 = load float, float addrspace(3)* %tmp52, align 4
+  %tmp54 = getelementptr inbounds float, float addrspace(3)* %arg, i32 31
+  %tmp55 = load float, float addrspace(3)* %tmp54, align 4
+  %tmp56 = tail call float @llvm.fmuladd.f32(float %tmp51, float %tmp53, float %tmp55)
+  %tmp57 = getelementptr inbounds float, float addrspace(3)* %arg, i32 33
+  %tmp58 = load float, float addrspace(3)* %tmp57, align 4
+  %tmp59 = getelementptr inbounds float, float addrspace(3)* %arg, i32 34
+  %tmp60 = load float, float addrspace(3)* %tmp59, align 4
+  %tmp61 = getelementptr inbounds float, float addrspace(3)* %arg, i32 35
+  %tmp62 = load float, float addrspace(3)* %tmp61, align 4
+  %tmp63 = tail call float @llvm.fmuladd.f32(float %tmp58, float %tmp60, float %tmp62)
+  %tmp64 = getelementptr inbounds float, float addrspace(3)* %arg, i32 37
+  %tmp65 = load float, float addrspace(3)* %tmp64, align 4
+  %tmp66 = getelementptr inbounds float, float addrspace(3)* %arg, i32 38
+  %tmp67 = load float, float addrspace(3)* %tmp66, align 4
+  %tmp68 = getelementptr inbounds float, float addrspace(3)* %arg, i32 39
+  %tmp69 = load float, float addrspace(3)* %tmp68, align 4
+  %tmp70 = tail call float @llvm.fmuladd.f32(float %tmp65, float %tmp67, float %tmp69)
+  %tmp71 = getelementptr inbounds float, float addrspace(3)* %arg, i32 41
+  %tmp72 = load float, float addrspace(3)* %tmp71, align 4
+  %tmp73 = getelementptr inbounds float, float addrspace(3)* %arg, i32 42
+  %tmp74 = load float, float addrspace(3)* %tmp73, align 4
+  %tmp75 = getelementptr inbounds float, float addrspace(3)* %arg, i32 43
+  %tmp76 = load float, float addrspace(3)* %tmp75, align 4
+  %tmp77 = tail call float @llvm.fmuladd.f32(float %tmp72, float %tmp74, float %tmp76)
+  %tmp78 = getelementptr inbounds float, float addrspace(3)* %arg, i32 45
+  %tmp79 = load float, float addrspace(3)* %tmp78, align 4
+  %tmp80 = getelementptr inbounds float, float addrspace(3)* %arg, i32 46
+  %tmp81 = load float, float addrspace(3)* %tmp80, align 4
+  %tmp82 = getelementptr inbounds float, float addrspace(3)* %arg, i32 47
+  %tmp83 = load float, float addrspace(3)* %tmp82, align 4
+  %tmp84 = tail call float @llvm.fmuladd.f32(float %tmp79, float %tmp81, float %tmp83)
+  %tmp85 = getelementptr inbounds float, float addrspace(3)* %arg, i32 49
+  %tmp86 = load float, float addrspace(3)* %tmp85, align 4
+  %tmp87 = getelementptr inbounds float, float addrspace(3)* %arg, i32 50
+  %tmp88 = load float, float addrspace(3)* %tmp87, align 4
+  %tmp89 = getelementptr inbounds float, float addrspace(3)* %arg, i32 51
+  %tmp90 = load float, float addrspace(3)* %tmp89, align 4
+  %tmp91 = tail call float @llvm.fmuladd.f32(float %tmp86, float %tmp88, float %tmp90)
+  %tmp92 = getelementptr inbounds float, float addrspace(3)* %arg, i32 53
+  %tmp93 = load float, float addrspace(3)* %tmp92, align 4
+  %tmp94 = getelementptr inbounds float, float addrspace(3)* %arg, i32 54
+  %tmp95 = load float, float addrspace(3)* %tmp94, align 4
+  %tmp96 = getelementptr inbounds float, float addrspace(3)* %arg, i32 55
+  %tmp97 = load float, float addrspace(3)* %tmp96, align 4
+  %tmp98 = tail call float @llvm.fmuladd.f32(float %tmp93, float %tmp95, float %tmp97)
+  %tmp99 = getelementptr inbounds float, float addrspace(3)* %arg, i32 57
+  %tmp100 = load float, float addrspace(3)* %tmp99, align 4
+  %tmp101 = getelementptr inbounds float, float addrspace(3)* %arg, i32 58
+  %tmp102 = load float, float addrspace(3)* %tmp101, align 4
+  %tmp103 = getelementptr inbounds float, float addrspace(3)* %arg, i32 59
+  %tmp104 = load float, float addrspace(3)* %tmp103, align 4
+  %tmp105 = tail call float @llvm.fmuladd.f32(float %tmp100, float %tmp102, float %tmp104)
+  %tmp106 = getelementptr inbounds float, float addrspace(3)* %arg, i32 61
+  %tmp107 = load float, float addrspace(3)* %tmp106, align 4
+  %tmp108 = getelementptr inbounds float, float addrspace(3)* %arg, i32 62
+  %tmp109 = load float, float addrspace(3)* %tmp108, align 4
+  %tmp110 = getelementptr inbounds float, float addrspace(3)* %arg, i32 63
+  %tmp111 = load float, float addrspace(3)* %tmp110, align 4
+  %tmp112 = tail call float @llvm.fmuladd.f32(float %tmp107, float %tmp109, float %tmp111)
+  %tmp113 = getelementptr inbounds float, float addrspace(3)* %arg, i32 65
+  %tmp114 = load float, float addrspace(3)* %tmp113, align 4
+  %tmp115 = getelementptr inbounds float, float addrspace(3)* %arg, i32 66
+  %tmp116 = load float, float addrspace(3)* %tmp115, align 4
+  %tmp117 = getelementptr inbounds float, float addrspace(3)* %arg, i32 67
+  %tmp118 = load float, float addrspace(3)* %tmp117, align 4
+  %tmp119 = tail call float @llvm.fmuladd.f32(float %tmp114, float %tmp116, float %tmp118)
+  %tmp120 = getelementptr inbounds float, float addrspace(3)* %arg, i32 69
+  %tmp121 = load float, float addrspace(3)* %tmp120, align 4
+  %tmp122 = getelementptr inbounds float, float addrspace(3)* %arg, i32 70
+  %tmp123 = load float, float addrspace(3)* %tmp122, align 4
+  %tmp124 = getelementptr inbounds float, float addrspace(3)* %arg, i32 71
+  %tmp125 = load float, float addrspace(3)* %tmp124, align 4
+  %tmp126 = tail call float @llvm.fmuladd.f32(float %tmp121, float %tmp123, float %tmp125)
+  %tmp127 = getelementptr inbounds float, float addrspace(3)* %arg, i32 73
+  %tmp128 = load float, float addrspace(3)* %tmp127, align 4
+  %tmp129 = getelementptr inbounds float, float addrspace(3)* %arg, i32 74
+  %tmp130 = load float, float addrspace(3)* %tmp129, align 4
+  %tmp131 = getelementptr inbounds float, float addrspace(3)* %arg, i32 75
+  %tmp132 = load float, float addrspace(3)* %tmp131, align 4
+  %tmp133 = tail call float @llvm.fmuladd.f32(float %tmp128, float %tmp130, float %tmp132)
+  %tmp134 = getelementptr inbounds float, float addrspace(3)* %arg, i32 77
+  %tmp135 = load float, float addrspace(3)* %tmp134, align 4
+  %tmp136 = getelementptr inbounds float, float addrspace(3)* %arg, i32 78
+  %tmp137 = load float, float addrspace(3)* %tmp136, align 4
+  %tmp138 = getelementptr inbounds float, float addrspace(3)* %arg, i32 79
+  %tmp139 = load float, float addrspace(3)* %tmp138, align 4
+  %tmp140 = tail call float @llvm.fmuladd.f32(float %tmp135, float %tmp137, float %tmp139)
+  %tmp141 = getelementptr inbounds float, float addrspace(3)* %arg, i32 81
+  %tmp142 = load float, float addrspace(3)* %tmp141, align 4
+  %tmp143 = getelementptr inbounds float, float addrspace(3)* %arg, i32 82
+  %tmp144 = load float, float addrspace(3)* %tmp143, align 4
+  %tmp145 = getelementptr inbounds float, float addrspace(3)* %arg, i32 83
+  %tmp146 = load float, float addrspace(3)* %tmp145, align 4
+  %tmp147 = tail call float @llvm.fmuladd.f32(float %tmp142, float %tmp144, float %tmp146)
+  %tmp148 = getelementptr inbounds float, float addrspace(3)* %arg, i32 85
+  %tmp149 = load float, float addrspace(3)* %tmp148, align 4
+  %tmp150 = getelementptr inbounds float, float addrspace(3)* %arg, i32 86
+  %tmp151 = load float, float addrspace(3)* %tmp150, align 4
+  %tmp152 = getelementptr inbounds float, float addrspace(3)* %arg, i32 87
+  %tmp153 = load float, float addrspace(3)* %tmp152, align 4
+  %tmp154 = tail call float @llvm.fmuladd.f32(float %tmp149, float %tmp151, float %tmp153)
+  %tmp155 = getelementptr inbounds float, float addrspace(3)* %arg, i32 89
+  %tmp156 = load float, float addrspace(3)* %tmp155, align 4
+  %tmp157 = getelementptr inbounds float, float addrspace(3)* %arg, i32 90
+  %tmp158 = load float, float addrspace(3)* %tmp157, align 4
+  %tmp159 = getelementptr inbounds float, float addrspace(3)* %arg, i32 91
+  %tmp160 = load float, float addrspace(3)* %tmp159, align 4
+  %tmp161 = tail call float @llvm.fmuladd.f32(float %tmp156, float %tmp158, float %tmp160)
+  %tmp162 = getelementptr inbounds float, float addrspace(3)* %arg, i32 93
+  %tmp163 = load float, float addrspace(3)* %tmp162, align 4
+  %tmp164 = getelementptr inbounds float, float addrspace(3)* %arg, i32 94
+  %tmp165 = load float, float addrspace(3)* %tmp164, align 4
+  %tmp166 = getelementptr inbounds float, float addrspace(3)* %arg, i32 95
+  %tmp167 = load float, float addrspace(3)* %tmp166, align 4
+  %tmp168 = tail call float @llvm.fmuladd.f32(float %tmp163, float %tmp165, float %tmp167)
+  %tmp169 = getelementptr inbounds float, float addrspace(3)* %arg, i32 97
+  %tmp170 = load float, float addrspace(3)* %tmp169, align 4
+  %tmp171 = getelementptr inbounds float, float addrspace(3)* %arg, i32 98
+  %tmp172 = load float, float addrspace(3)* %tmp171, align 4
+  %tmp173 = getelementptr inbounds float, float addrspace(3)* %arg, i32 99
+  %tmp174 = load float, float addrspace(3)* %tmp173, align 4
+  %tmp175 = tail call float @llvm.fmuladd.f32(float %tmp170, float %tmp172, float %tmp174)
+  %tmp176 = getelementptr inbounds float, float addrspace(3)* %arg, i32 101
+  %tmp177 = load float, float addrspace(3)* %tmp176, align 4
+  %tmp178 = getelementptr inbounds float, float addrspace(3)* %arg, i32 102
+  %tmp179 = load float, float addrspace(3)* %tmp178, align 4
+  %tmp180 = getelementptr inbounds float, float addrspace(3)* %arg, i32 103
+  %tmp181 = load float, float addrspace(3)* %tmp180, align 4
+  %tmp182 = tail call float @llvm.fmuladd.f32(float %tmp177, float %tmp179, float %tmp181)
+  %tmp183 = getelementptr inbounds float, float addrspace(3)* %arg, i32 105
+  %tmp184 = load float, float addrspace(3)* %tmp183, align 4
+  %tmp185 = getelementptr inbounds float, float addrspace(3)* %arg, i32 106
+  %tmp186 = load float, float addrspace(3)* %tmp185, align 4
+  %tmp187 = getelementptr inbounds float, float addrspace(3)* %arg, i32 107
+  %tmp188 = load float, float addrspace(3)* %tmp187, align 4
+  %tmp189 = tail call float @llvm.fmuladd.f32(float %tmp184, float %tmp186, float %tmp188)
+  %tmp190 = getelementptr inbounds float, float addrspace(3)* %arg, i32 109
+  %tmp191 = load float, float addrspace(3)* %tmp190, align 4
+  %tmp192 = getelementptr inbounds float, float addrspace(3)* %arg, i32 110
+  %tmp193 = load float, float addrspace(3)* %tmp192, align 4
+  %tmp194 = getelementptr inbounds float, float addrspace(3)* %arg, i32 111
+  %tmp195 = load float, float addrspace(3)* %tmp194, align 4
+  %tmp196 = tail call float @llvm.fmuladd.f32(float %tmp191, float %tmp193, float %tmp195)
+  %tmp197 = getelementptr inbounds float, float addrspace(3)* %arg, i32 113
+  %tmp198 = load float, float addrspace(3)* %tmp197, align 4
+  %tmp199 = getelementptr inbounds float, float addrspace(3)* %arg, i32 114
+  %tmp200 = load float, float addrspace(3)* %tmp199, align 4
+  %tmp201 = getelementptr inbounds float, float addrspace(3)* %arg, i32 115
+  %tmp202 = load float, float addrspace(3)* %tmp201, align 4
+  %tmp203 = tail call float @llvm.fmuladd.f32(float %tmp198, float %tmp200, float %tmp202)
+  %tmp204 = getelementptr inbounds float, float addrspace(3)* %arg, i32 117
+  %tmp205 = load float, float addrspace(3)* %tmp204, align 4
+  %tmp206 = getelementptr inbounds float, float addrspace(3)* %arg, i32 118
+  %tmp207 = load float, float addrspace(3)* %tmp206, align 4
+  %tmp208 = getelementptr inbounds float, float addrspace(3)* %arg, i32 119
+  %tmp209 = load float, float addrspace(3)* %tmp208, align 4
+  %tmp210 = tail call float @llvm.fmuladd.f32(float %tmp205, float %tmp207, float %tmp209)
+  %tmp211 = getelementptr inbounds float, float addrspace(3)* %arg, i32 121
+  %tmp212 = load float, float addrspace(3)* %tmp211, align 4
+  %tmp213 = getelementptr inbounds float, float addrspace(3)* %arg, i32 122
+  %tmp214 = load float, float addrspace(3)* %tmp213, align 4
+  %tmp215 = getelementptr inbounds float, float addrspace(3)* %arg, i32 123
+  %tmp216 = load float, float addrspace(3)* %tmp215, align 4
+  %tmp217 = tail call float @llvm.fmuladd.f32(float %tmp212, float %tmp214, float %tmp216)
+  %tmp218 = getelementptr inbounds float, float addrspace(3)* %arg, i32 125
+  %tmp219 = load float, float addrspace(3)* %tmp218, align 4
+  %tmp220 = getelementptr inbounds float, float addrspace(3)* %arg, i32 126
+  %tmp221 = load float, float addrspace(3)* %tmp220, align 4
+  %tmp222 = getelementptr inbounds float, float addrspace(3)* %arg, i32 127
+  %tmp223 = load float, float addrspace(3)* %tmp222, align 4
+  %tmp224 = tail call float @llvm.fmuladd.f32(float %tmp219, float %tmp221, float %tmp223)
+  %tmp225 = getelementptr inbounds float, float addrspace(3)* %arg, i32 129
+  %tmp226 = load float, float addrspace(3)* %tmp225, align 4
+  %tmp227 = getelementptr inbounds float, float addrspace(3)* %arg, i32 130
+  %tmp228 = load float, float addrspace(3)* %tmp227, align 4
+  %tmp229 = getelementptr inbounds float, float addrspace(3)* %arg, i32 131
+  %tmp230 = load float, float addrspace(3)* %tmp229, align 4
+  %tmp231 = tail call float @llvm.fmuladd.f32(float %tmp226, float %tmp228, float %tmp230)
+  %tmp232 = getelementptr inbounds float, float addrspace(3)* %arg, i32 133
+  %tmp233 = load float, float addrspace(3)* %tmp232, align 4
+  %tmp234 = getelementptr inbounds float, float addrspace(3)* %arg, i32 134
+  %tmp235 = load float, float addrspace(3)* %tmp234, align 4
+  %tmp236 = getelementptr inbounds float, float addrspace(3)* %arg, i32 135
+  %tmp237 = load float, float addrspace(3)* %tmp236, align 4
+  %tmp238 = tail call float @llvm.fmuladd.f32(float %tmp233, float %tmp235, float %tmp237)
+  %tmp239 = getelementptr inbounds float, float addrspace(3)* %arg, i32 137
+  %tmp240 = load float, float addrspace(3)* %tmp239, align 4
+  %tmp241 = getelementptr inbounds float, float addrspace(3)* %arg, i32 138
+  %tmp242 = load float, float addrspace(3)* %tmp241, align 4
+  %tmp243 = getelementptr inbounds float, float addrspace(3)* %arg, i32 139
+  %tmp244 = load float, float addrspace(3)* %tmp243, align 4
+  %tmp245 = tail call float @llvm.fmuladd.f32(float %tmp240, float %tmp242, float %tmp244)
+  %tmp246 = getelementptr inbounds float, float addrspace(3)* %arg, i32 141
+  %tmp247 = load float, float addrspace(3)* %tmp246, align 4
+  %tmp248 = getelementptr inbounds float, float addrspace(3)* %arg, i32 142
+  %tmp249 = load float, float addrspace(3)* %tmp248, align 4
+  %tmp250 = getelementptr inbounds float, float addrspace(3)* %arg, i32 143
+  %tmp251 = load float, float addrspace(3)* %tmp250, align 4
+  %tmp252 = tail call float @llvm.fmuladd.f32(float %tmp247, float %tmp249, float %tmp251)
+  %tmp253 = getelementptr inbounds float, float addrspace(3)* %arg, i32 145
+  %tmp254 = load float, float addrspace(3)* %tmp253, align 4
+  %tmp255 = getelementptr inbounds float, float addrspace(3)* %arg, i32 146
+  %tmp256 = load float, float addrspace(3)* %tmp255, align 4
+  %tmp257 = getelementptr inbounds float, float addrspace(3)* %arg, i32 147
+  %tmp258 = load float, float addrspace(3)* %tmp257, align 4
+  %tmp259 = tail call float @llvm.fmuladd.f32(float %tmp254, float %tmp256, float %tmp258)
+  %tmp260 = getelementptr inbounds float, float addrspace(3)* %arg, i32 149
+  %tmp261 = load float, float addrspace(3)* %tmp260, align 4
+  %tmp262 = getelementptr inbounds float, float addrspace(3)* %arg, i32 150
+  %tmp263 = load float, float addrspace(3)* %tmp262, align 4
+  %tmp264 = getelementptr inbounds float, float addrspace(3)* %arg, i32 151
+  %tmp265 = load float, float addrspace(3)* %tmp264, align 4
+  %tmp266 = tail call float @llvm.fmuladd.f32(float %tmp261, float %tmp263, float %tmp265)
+  %tmp267 = getelementptr inbounds float, float addrspace(3)* %arg, i32 153
+  %tmp268 = load float, float addrspace(3)* %tmp267, align 4
+  %tmp269 = getelementptr inbounds float, float addrspace(3)* %arg, i32 154
+  %tmp270 = load float, float addrspace(3)* %tmp269, align 4
+  %tmp271 = getelementptr inbounds float, float addrspace(3)* %arg, i32 155
+  %tmp272 = load float, float addrspace(3)* %tmp271, align 4
+  %tmp273 = tail call float @llvm.fmuladd.f32(float %tmp268, float %tmp270, float %tmp272)
+  %tmp274 = getelementptr inbounds float, float addrspace(3)* %arg, i32 157
+  %tmp275 = load float, float addrspace(3)* %tmp274, align 4
+  %tmp276 = getelementptr inbounds float, float addrspace(3)* %arg, i32 158
+  %tmp277 = load float, float addrspace(3)* %tmp276, align 4
+  %tmp278 = getelementptr inbounds float, float addrspace(3)* %arg, i32 159
+  %tmp279 = load float, float addrspace(3)* %tmp278, align 4
+  %tmp280 = tail call float @llvm.fmuladd.f32(float %tmp275, float %tmp277, float %tmp279)
+  %tmp281 = getelementptr inbounds float, float addrspace(3)* %arg, i32 161
+  %tmp282 = load float, float addrspace(3)* %tmp281, align 4
+  %tmp283 = getelementptr inbounds float, float addrspace(3)* %arg, i32 162
+  %tmp284 = load float, float addrspace(3)* %tmp283, align 4
+  %tmp285 = getelementptr inbounds float, float addrspace(3)* %arg, i32 163
+  %tmp286 = load float, float addrspace(3)* %tmp285, align 4
+  %tmp287 = tail call float @llvm.fmuladd.f32(float %tmp282, float %tmp284, float %tmp286)
+  %tmp288 = getelementptr inbounds float, float addrspace(3)* %arg, i32 165
+  %tmp289 = load float, float addrspace(3)* %tmp288, align 4
+  %tmp290 = getelementptr inbounds float, float addrspace(3)* %arg, i32 166
+  %tmp291 = load float, float addrspace(3)* %tmp290, align 4
+  %tmp292 = getelementptr inbounds float, float addrspace(3)* %arg, i32 167
+  %tmp293 = load float, float addrspace(3)* %tmp292, align 4
+  %tmp294 = tail call float @llvm.fmuladd.f32(float %tmp289, float %tmp291, float %tmp293)
+  %tmp295 = getelementptr inbounds float, float addrspace(3)* %arg, i32 169
+  %tmp296 = load float, float addrspace(3)* %tmp295, align 4
+  %tmp297 = getelementptr inbounds float, float addrspace(3)* %arg, i32 170
+  %tmp298 = load float, float addrspace(3)* %tmp297, align 4
+  %tmp299 = getelementptr inbounds float, float addrspace(3)* %arg, i32 171
+  %tmp300 = load float, float addrspace(3)* %tmp299, align 4
+  %tmp301 = tail call float @llvm.fmuladd.f32(float %tmp296, float %tmp298, float %tmp300)
+  %tmp302 = getelementptr inbounds float, float addrspace(3)* %arg, i32 173
+  %tmp303 = load float, float addrspace(3)* %tmp302, align 4
+  %tmp304 = getelementptr inbounds float, float addrspace(3)* %arg, i32 174
+  %tmp305 = load float, float addrspace(3)* %tmp304, align 4
+  %tmp306 = getelementptr inbounds float, float addrspace(3)* %arg, i32 175
+  %tmp307 = load float, float addrspace(3)* %tmp306, align 4
+  %tmp308 = tail call float @llvm.fmuladd.f32(float %tmp303, float %tmp305, float %tmp307)
+  %tmp309 = getelementptr inbounds float, float addrspace(3)* %arg, i32 177
+  %tmp310 = load float, float addrspace(3)* %tmp309, align 4
+  %tmp311 = getelementptr inbounds float, float addrspace(3)* %arg, i32 178
+  %tmp312 = load float, float addrspace(3)* %tmp311, align 4
+  %tmp313 = getelementptr inbounds float, float addrspace(3)* %arg, i32 179
+  %tmp314 = load float, float addrspace(3)* %tmp313, align 4
+  %tmp315 = tail call float @llvm.fmuladd.f32(float %tmp310, float %tmp312, float %tmp314)
+  %tmp316 = getelementptr inbounds float, float addrspace(3)* %arg, i32 181
+  %tmp317 = load float, float addrspace(3)* %tmp316, align 4
+  %tmp318 = getelementptr inbounds float, float addrspace(3)* %arg, i32 182
+  %tmp319 = load float, float addrspace(3)* %tmp318, align 4
+  %tmp320 = getelementptr inbounds float, float addrspace(3)* %arg, i32 183
+  %tmp321 = load float, float addrspace(3)* %tmp320, align 4
+  %tmp322 = tail call float @llvm.fmuladd.f32(float %tmp317, float %tmp319, float %tmp321)
+  %tmp323 = getelementptr inbounds float, float addrspace(3)* %arg, i32 185
+  %tmp324 = load float, float addrspace(3)* %tmp323, align 4
+  %tmp325 = getelementptr inbounds float, float addrspace(3)* %arg, i32 186
+  %tmp326 = load float, float addrspace(3)* %tmp325, align 4
+  %tmp327 = getelementptr inbounds float, float addrspace(3)* %arg, i32 187
+  %tmp328 = load float, float addrspace(3)* %tmp327, align 4
+  %tmp329 = tail call float @llvm.fmuladd.f32(float %tmp324, float %tmp326, float %tmp328)
+  %tmp330 = getelementptr inbounds float, float addrspace(3)* %arg, i32 189
+  %tmp331 = load float, float addrspace(3)* %tmp330, align 4
+  %tmp332 = getelementptr inbounds float, float addrspace(3)* %arg, i32 190
+  %tmp333 = load float, float addrspace(3)* %tmp332, align 4
+  %tmp334 = getelementptr inbounds float, float addrspace(3)* %arg, i32 191
+  %tmp335 = load float, float addrspace(3)* %tmp334, align 4
+  %tmp336 = tail call float @llvm.fmuladd.f32(float %tmp331, float %tmp333, float %tmp335)
+  %tmp337 = getelementptr inbounds float, float addrspace(3)* %arg, i32 193
+  %tmp338 = load float, float addrspace(3)* %tmp337, align 4
+  %tmp339 = getelementptr inbounds float, float addrspace(3)* %arg, i32 194
+  %tmp340 = load float, float addrspace(3)* %tmp339, align 4
+  %tmp341 = getelementptr inbounds float, float addrspace(3)* %arg, i32 195
+  %tmp342 = load float, float addrspace(3)* %tmp341, align 4
+  %tmp343 = tail call float @llvm.fmuladd.f32(float %tmp338, float %tmp340, float %tmp342)
+  %tmp344 = getelementptr inbounds float, float addrspace(3)* %arg, i32 197
+  %tmp345 = load float, float addrspace(3)* %tmp344, align 4
+  %tmp346 = getelementptr inbounds float, float addrspace(3)* %arg, i32 198
+  %tmp347 = load float, float addrspace(3)* %tmp346, align 4
+  %tmp348 = getelementptr inbounds float, float addrspace(3)* %arg, i32 199
+  %tmp349 = load float, float addrspace(3)* %tmp348, align 4
+  %tmp350 = tail call float @llvm.fmuladd.f32(float %tmp345, float %tmp347, float %tmp349)
+  %tmp351 = getelementptr inbounds float, float addrspace(3)* %arg, i32 201
+  %tmp352 = load float, float addrspace(3)* %tmp351, align 4
+  %tmp353 = getelementptr inbounds float, float addrspace(3)* %arg, i32 202
+  %tmp354 = load float, float addrspace(3)* %tmp353, align 4
+  %tmp355 = getelementptr inbounds float, float addrspace(3)* %arg, i32 203
+  %tmp356 = load float, float addrspace(3)* %tmp355, align 4
+  %tmp357 = tail call float @llvm.fmuladd.f32(float %tmp352, float %tmp354, float %tmp356)
+  %tmp358 = getelementptr inbounds float, float addrspace(3)* %arg, i32 205
+  %tmp359 = load float, float addrspace(3)* %tmp358, align 4
+  %tmp360 = getelementptr inbounds float, float addrspace(3)* %arg, i32 206
+  %tmp361 = load float, float addrspace(3)* %tmp360, align 4
+  %tmp362 = getelementptr inbounds float, float addrspace(3)* %arg, i32 207
+  %tmp363 = load float, float addrspace(3)* %tmp362, align 4
+  %tmp364 = tail call float @llvm.fmuladd.f32(float %tmp359, float %tmp361, float %tmp363)
+  %tmp365 = getelementptr inbounds float, float addrspace(3)* %arg, i32 209
+  %tmp366 = load float, float addrspace(3)* %tmp365, align 4
+  %tmp367 = getelementptr inbounds float, float addrspace(3)* %arg, i32 210
+  %tmp368 = load float, float addrspace(3)* %tmp367, align 4
+  %tmp369 = getelementptr inbounds float, float addrspace(3)* %arg, i32 211
+  %tmp370 = load float, float addrspace(3)* %tmp369, align 4
+  %tmp371 = tail call float @llvm.fmuladd.f32(float %tmp366, float %tmp368, float %tmp370)
+  %tmp372 = getelementptr inbounds float, float addrspace(3)* %arg, i32 213
+  %tmp373 = load float, float addrspace(3)* %tmp372, align 4
+  %tmp374 = getelementptr inbounds float, float addrspace(3)* %arg, i32 214
+  %tmp375 = load float, float addrspace(3)* %tmp374, align 4
+  %tmp376 = getelementptr inbounds float, float addrspace(3)* %arg, i32 215
+  %tmp377 = load float, float addrspace(3)* %tmp376, align 4
+  %tmp378 = tail call float @llvm.fmuladd.f32(float %tmp373, float %tmp375, float %tmp377)
+  %tmp379 = getelementptr inbounds float, float addrspace(3)* %arg, i32 217
+  %tmp380 = load float, float addrspace(3)* %tmp379, align 4
+  %tmp381 = getelementptr inbounds float, float addrspace(3)* %arg, i32 218
+  %tmp382 = load float, float addrspace(3)* %tmp381, align 4
+  %tmp383 = getelementptr inbounds float, float addrspace(3)* %arg, i32 219
+  %tmp384 = load float, float addrspace(3)* %tmp383, align 4
+  %tmp385 = tail call float @llvm.fmuladd.f32(float %tmp380, float %tmp382, float %tmp384)
+  %tmp386 = getelementptr inbounds float, float addrspace(3)* %arg, i32 221
+  %tmp387 = load float, float addrspace(3)* %tmp386, align 4
+  %tmp388 = getelementptr inbounds float, float addrspace(3)* %arg, i32 222
+  %tmp389 = load float, float addrspace(3)* %tmp388, align 4
+  %tmp390 = getelementptr inbounds float, float addrspace(3)* %arg, i32 223
+  %tmp391 = load float, float addrspace(3)* %tmp390, align 4
+  %tmp392 = tail call float @llvm.fmuladd.f32(float %tmp387, float %tmp389, float %tmp391)
+  %tmp393 = getelementptr inbounds float, float addrspace(3)* %arg, i32 225
+  %tmp394 = load float, float addrspace(3)* %tmp393, align 4
+  %tmp395 = getelementptr inbounds float, float addrspace(3)* %arg, i32 226
+  %tmp396 = load float, float addrspace(3)* %tmp395, align 4
+  %tmp397 = getelementptr inbounds float, float addrspace(3)* %arg, i32 227
+  %tmp398 = load float, float addrspace(3)* %tmp397, align 4
+  %tmp399 = tail call float @llvm.fmuladd.f32(float %tmp394, float %tmp396, float %tmp398)
+  %tmp400 = getelementptr inbounds float, float addrspace(3)* %arg, i32 229
+  %tmp401 = load float, float addrspace(3)* %tmp400, align 4
+  %tmp402 = getelementptr inbounds float, float addrspace(3)* %arg, i32 230
+  %tmp403 = load float, float addrspace(3)* %tmp402, align 4
+  %tmp404 = getelementptr inbounds float, float addrspace(3)* %arg, i32 231
+  %tmp405 = load float, float addrspace(3)* %tmp404, align 4
+  %tmp406 = tail call float @llvm.fmuladd.f32(float %tmp401, float %tmp403, float %tmp405)
+  %tmp407 = getelementptr inbounds float, float addrspace(3)* %arg, i32 233
+  %tmp408 = load float, float addrspace(3)* %tmp407, align 4
+  %tmp409 = getelementptr inbounds float, float addrspace(3)* %arg, i32 234
+  %tmp410 = load float, float addrspace(3)* %tmp409, align 4
+  %tmp411 = getelementptr inbounds float, float addrspace(3)* %arg, i32 235
+  %tmp412 = load float, float addrspace(3)* %tmp411, align 4
+  %tmp413 = tail call float @llvm.fmuladd.f32(float %tmp408, float %tmp410, float %tmp412)
+  %tmp414 = getelementptr inbounds float, float addrspace(3)* %arg, i32 237
+  %tmp415 = load float, float addrspace(3)* %tmp414, align 4
+  %tmp416 = getelementptr inbounds float, float addrspace(3)* %arg, i32 238
+  %tmp417 = load float, float addrspace(3)* %tmp416, align 4
+  %tmp418 = getelementptr inbounds float, float addrspace(3)* %arg, i32 239
+  %tmp419 = load float, float addrspace(3)* %tmp418, align 4
+  %tmp420 = tail call float @llvm.fmuladd.f32(float %tmp415, float %tmp417, float %tmp419)
+  %tmp421 = getelementptr inbounds float, float addrspace(3)* %arg, i32 241
+  %tmp422 = load float, float addrspace(3)* %tmp421, align 4
+  %tmp423 = getelementptr inbounds float, float addrspace(3)* %arg, i32 242
+  %tmp424 = load float, float addrspace(3)* %tmp423, align 4
+  %tmp425 = getelementptr inbounds float, float addrspace(3)* %arg, i32 243
+  %tmp426 = load float, float addrspace(3)* %tmp425, align 4
+  %tmp427 = tail call float @llvm.fmuladd.f32(float %tmp422, float %tmp424, float %tmp426)
+  %tmp428 = getelementptr inbounds float, float addrspace(3)* %arg, i32 245
+  %tmp429 = load float, float addrspace(3)* %tmp428, align 4
+  %tmp430 = getelementptr inbounds float, float addrspace(3)* %arg, i32 246
+  %tmp431 = load float, float addrspace(3)* %tmp430, align 4
+  %tmp432 = getelementptr inbounds float, float addrspace(3)* %arg, i32 247
+  %tmp433 = load float, float addrspace(3)* %tmp432, align 4
+  %tmp434 = tail call float @llvm.fmuladd.f32(float %tmp429, float %tmp431, float %tmp433)
+  %tmp435 = getelementptr inbounds float, float addrspace(3)* %arg, i32 249
+  %tmp436 = load float, float addrspace(3)* %tmp435, align 4
+  %tmp437 = getelementptr inbounds float, float addrspace(3)* %arg, i32 250
+  %tmp438 = load float, float addrspace(3)* %tmp437, align 4
+  %tmp439 = getelementptr inbounds float, float addrspace(3)* %arg, i32 251
+  %tmp440 = load float, float addrspace(3)* %tmp439, align 4
+  %tmp441 = tail call float @llvm.fmuladd.f32(float %tmp436, float %tmp438, float %tmp440)
+  %tmp442 = getelementptr inbounds float, float addrspace(3)* %arg, i32 253
+  %tmp443 = load float, float addrspace(3)* %tmp442, align 4
+  %tmp444 = getelementptr inbounds float, float addrspace(3)* %arg, i32 254
+  %tmp445 = load float, float addrspace(3)* %tmp444, align 4
+  %tmp446 = getelementptr inbounds float, float addrspace(3)* %arg, i32 255
+  %tmp447 = load float, float addrspace(3)* %tmp446, align 4
+  %tmp448 = tail call float @llvm.fmuladd.f32(float %tmp443, float %tmp445, float %tmp447)
+  store float %tmp7, float addrspace(1)* %arg1, align 4
+  %tmp449 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 1
+  store float %tmp14, float addrspace(1)* %tmp449, align 4
+  %tmp450 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 2
+  store float %tmp21, float addrspace(1)* %tmp450, align 4
+  %tmp451 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 3
+  store float %tmp28, float addrspace(1)* %tmp451, align 4
+  %tmp452 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 4
+  store float %tmp35, float addrspace(1)* %tmp452, align 4
+  %tmp453 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 5
+  store float %tmp42, float addrspace(1)* %tmp453, align 4
+  %tmp454 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 6
+  store float %tmp49, float addrspace(1)* %tmp454, align 4
+  %tmp455 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 7
+  store float %tmp56, float addrspace(1)* %tmp455, align 4
+  %tmp456 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 8
+  store float %tmp63, float addrspace(1)* %tmp456, align 4
+  %tmp457 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 9
+  store float %tmp70, float addrspace(1)* %tmp457, align 4
+  %tmp458 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 10
+  store float %tmp77, float addrspace(1)* %tmp458, align 4
+  %tmp459 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 11
+  store float %tmp84, float addrspace(1)* %tmp459, align 4
+  %tmp460 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 12
+  store float %tmp91, float addrspace(1)* %tmp460, align 4
+  %tmp461 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 13
+  store float %tmp98, float addrspace(1)* %tmp461, align 4
+  %tmp462 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 14
+  store float %tmp105, float addrspace(1)* %tmp462, align 4
+  %tmp463 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 15
+  store float %tmp112, float addrspace(1)* %tmp463, align 4
+  %tmp464 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 16
+  store float %tmp119, float addrspace(1)* %tmp464, align 4
+  %tmp465 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 17
+  store float %tmp126, float addrspace(1)* %tmp465, align 4
+  %tmp466 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 18
+  store float %tmp133, float addrspace(1)* %tmp466, align 4
+  %tmp467 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 19
+  store float %tmp140, float addrspace(1)* %tmp467, align 4
+  %tmp468 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 20
+  store float %tmp147, float addrspace(1)* %tmp468, align 4
+  %tmp469 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 21
+  store float %tmp154, float addrspace(1)* %tmp469, align 4
+  %tmp470 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 22
+  store float %tmp161, float addrspace(1)* %tmp470, align 4
+  %tmp471 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 23
+  store float %tmp168, float addrspace(1)* %tmp471, align 4
+  %tmp472 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 24
+  store float %tmp175, float addrspace(1)* %tmp472, align 4
+  %tmp473 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 25
+  store float %tmp182, float addrspace(1)* %tmp473, align 4
+  %tmp474 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 26
+  store float %tmp189, float addrspace(1)* %tmp474, align 4
+  %tmp475 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 27
+  store float %tmp196, float addrspace(1)* %tmp475, align 4
+  %tmp476 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 28
+  store float %tmp203, float addrspace(1)* %tmp476, align 4
+  %tmp477 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 29
+  store float %tmp210, float addrspace(1)* %tmp477, align 4
+  %tmp478 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 30
+  store float %tmp217, float addrspace(1)* %tmp478, align 4
+  %tmp479 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 31
+  store float %tmp224, float addrspace(1)* %tmp479, align 4
+  %tmp480 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 32
+  store float %tmp231, float addrspace(1)* %tmp480, align 4
+  %tmp481 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 33
+  store float %tmp238, float addrspace(1)* %tmp481, align 4
+  %tmp482 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 34
+  store float %tmp245, float addrspace(1)* %tmp482, align 4
+  %tmp483 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 35
+  store float %tmp252, float addrspace(1)* %tmp483, align 4
+  %tmp484 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 36
+  store float %tmp259, float addrspace(1)* %tmp484, align 4
+  %tmp485 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 37
+  store float %tmp266, float addrspace(1)* %tmp485, align 4
+  %tmp486 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 38
+  store float %tmp273, float addrspace(1)* %tmp486, align 4
+  %tmp487 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 39
+  store float %tmp280, float addrspace(1)* %tmp487, align 4
+  %tmp488 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 40
+  store float %tmp287, float addrspace(1)* %tmp488, align 4
+  %tmp489 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 41
+  store float %tmp294, float addrspace(1)* %tmp489, align 4
+  %tmp490 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 42
+  store float %tmp301, float addrspace(1)* %tmp490, align 4
+  %tmp491 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 43
+  store float %tmp308, float addrspace(1)* %tmp491, align 4
+  %tmp492 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 44
+  store float %tmp315, float addrspace(1)* %tmp492, align 4
+  %tmp493 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 45
+  store float %tmp322, float addrspace(1)* %tmp493, align 4
+  %tmp494 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 46
+  store float %tmp329, float addrspace(1)* %tmp494, align 4
+  %tmp495 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 47
+  store float %tmp336, float addrspace(1)* %tmp495, align 4
+  %tmp496 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 48
+  store float %tmp343, float addrspace(1)* %tmp496, align 4
+  %tmp497 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 49
+  store float %tmp350, float addrspace(1)* %tmp497, align 4
+  %tmp498 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 50
+  store float %tmp357, float addrspace(1)* %tmp498, align 4
+  %tmp499 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 51
+  store float %tmp364, float addrspace(1)* %tmp499, align 4
+  %tmp500 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 52
+  store float %tmp371, float addrspace(1)* %tmp500, align 4
+  %tmp501 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 53
+  store float %tmp378, float addrspace(1)* %tmp501, align 4
+  %tmp502 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 54
+  store float %tmp385, float addrspace(1)* %tmp502, align 4
+  %tmp503 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 55
+  store float %tmp392, float addrspace(1)* %tmp503, align 4
+  %tmp504 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 56
+  store float %tmp399, float addrspace(1)* %tmp504, align 4
+  %tmp505 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 57
+  store float %tmp406, float addrspace(1)* %tmp505, align 4
+  %tmp506 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 58
+  store float %tmp413, float addrspace(1)* %tmp506, align 4
+  %tmp507 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 59
+  store float %tmp420, float addrspace(1)* %tmp507, align 4
+  %tmp508 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 60
+  store float %tmp427, float addrspace(1)* %tmp508, align 4
+  %tmp509 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 61
+  store float %tmp434, float addrspace(1)* %tmp509, align 4
+  %tmp510 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 62
+  store float %tmp441, float addrspace(1)* %tmp510, align 4
+  %tmp511 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 63
+  store float %tmp448, float addrspace(1)* %tmp511, align 4
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare float @llvm.fmuladd.f32(float, float, float) #1
+
+attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll b/test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll
index 5c47c163dcce0..118bb9aa66e8f 100644
--- a/test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll
+++ b/test/CodeGen/AMDGPU/schedule-kernel-arg-loads.ll
@@ -1,17 +1,15 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=SI -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=SI -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=VI -check-prefix=GCN %s
 
 ; FUNC-LABEL: {{^}}cluster_arg_loads:
-; FIXME: Due to changes in the load clustering heuristics.  We no longer
-;        cluster all argument loads together on SI.
-; SI: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0xd
 ; SI: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
 ; SI-NEXT: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xb
+; SI-NEXT: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0xd
 ; SI-NEXT: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0xe
-; VI: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0x34
 ; VI: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; VI: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; VI: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0x38
+; VI-NEXT: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; VI-NEXT: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0x34
+; VI-NEXT: s_load_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, 0x38
 define amdgpu_kernel void @cluster_arg_loads(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 %x, i32 %y) nounwind {
   store i32 %x, i32 addrspace(1)* %out0, align 4
   store i32 %y, i32 addrspace(1)* %out1, align 4
diff --git a/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll b/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
index 0d19c1e6a8f34..2072cf5e88731 100644
--- a/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
+++ b/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -march=amdgcn -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -misched=gcn-minreg -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -misched=gcn-max-occupancy-experimental -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
 
@@ -7,7 +7,7 @@
 ; SI: NumVgprs: {{[1-9]$}}
 
 ; stores may alias loads
-; VI: NumSgprs: {{[1-5][0-9]$}}
+; VI: NumSgprs: {{[0-9]$}}
 ; VI: NumVgprs: {{[1-3][0-9]$}}
 
 define amdgpu_kernel void @load_fma_store(float addrspace(3)* nocapture readonly %in_arg, float addrspace(1)* nocapture %out_arg) {
diff --git a/test/CodeGen/AMDGPU/schedule-regpressure.mir b/test/CodeGen/AMDGPU/schedule-regpressure.mir
index 3a20ec732e5bf..afc2fab08f870 100644
--- a/test/CodeGen/AMDGPU/schedule-regpressure.mir
+++ b/test/CodeGen/AMDGPU/schedule-regpressure.mir
@@ -4,7 +4,7 @@
 # Check there is no SReg_32 pressure created by DS_* instructions because of M0 use
 
 # CHECK: ScheduleDAGMILive::schedule starting
-# CHECK: SU({{.*}} = DS_READ_B32 {{.*}} %M0<imp-use>, %EXEC<imp-use>
+# CHECK: SU({{.*}} = DS_READ_B32 {{.*}} implicit %m0, implicit %exec
 # CHECK: Pressure Diff : {{$}}
 # CHECK: SU({{.*}} DS_WRITE_B32
 
diff --git a/test/CodeGen/AMDGPU/scratch-buffer.ll b/test/CodeGen/AMDGPU/scratch-buffer.ll
index 4ae9871865f5e..ade5a96929a8e 100644
--- a/test/CodeGen/AMDGPU/scratch-buffer.ll
+++ b/test/CodeGen/AMDGPU/scratch-buffer.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -verify-machineinstrs -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -march=amdgcn < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; When a frame index offset is more than 12-bits, make sure we don't store
 ; it in mubuf's offset field.
@@ -50,7 +50,7 @@ done:
 ; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
 ; This constant isn't folded, because it has multiple uses.
 ; GCN-DAG: v_mov_b32_e32 [[K8000:v[0-9]+]], 0x8004
-; GCN-DAG: v_add_i32_e32 [[OFFSET:v[0-9]+]], vcc, [[K8000]]
+; GCN-DAG: v_add_{{[iu]}}32_e32 [[OFFSET:v[0-9]+]], vcc, [[K8000]]
 ; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}}
 
 define amdgpu_kernel void @legal_offset_fi_offset(i32 addrspace(1)* %out, i32 %cond, i32 addrspace(1)* %offsets, i32 %if_offset, i32 %else_offset) {
@@ -86,8 +86,21 @@ done:
   ret void
 }
 
+; GCN-LABEL: {{^}}neg_vaddr_offset_inbounds:
+; GCN: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 16, v{{[0-9]+}}
+; GCN: buffer_store_dword v{{[0-9]+}}, [[ADD]], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen{{$}}
+define amdgpu_kernel void @neg_vaddr_offset_inbounds(i32 %offset) {
+entry:
+  %array = alloca [8192 x i32]
+  %ptr_offset = add i32 %offset, 4
+  %ptr = getelementptr inbounds [8192 x i32], [8192 x i32]* %array, i32 0, i32 %ptr_offset
+  store i32 0, i32* %ptr
+  ret void
+}
+
 ; GCN-LABEL: {{^}}neg_vaddr_offset:
-; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:16{{$}}
+; GCN: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 16, v{{[0-9]+}}
+; GCN: buffer_store_dword v{{[0-9]+}}, [[ADD]], s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen{{$}}
 define amdgpu_kernel void @neg_vaddr_offset(i32 %offset) {
 entry:
   %array = alloca [8192 x i32]
diff --git a/test/CodeGen/AMDGPU/scratch-simple.ll b/test/CodeGen/AMDGPU/scratch-simple.ll
index 5e0178072e5e2..238bcc54928f6 100644
--- a/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=verde -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=gfx804 -mattr=-flat-for-global -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx803 -mattr=-flat-for-global -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
 
 ; This used to fail due to a v_add_i32 instruction with an illegal immediate
 ; operand that was created during Local Stack Slot Allocation. Test case derived
diff --git a/test/CodeGen/AMDGPU/sdiv.ll b/test/CodeGen/AMDGPU/sdiv.ll
index 305107f690fb8..b79bca54bbdd3 100644
--- a/test/CodeGen/AMDGPU/sdiv.ll
+++ b/test/CodeGen/AMDGPU/sdiv.ll
@@ -1,5 +1,6 @@
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 ; The code generated by sdiv is long and complex and may frequently change.
@@ -37,10 +38,10 @@ define amdgpu_kernel void @sdiv_i32_4(i32 addrspace(1)* %out, i32 addrspace(1)*
 ; SI-DAG: buffer_load_dword [[VAL:v[0-9]+]],
 ; SI-DAG: v_mov_b32_e32 [[MAGIC:v[0-9]+]], 0x98a1930b
 ; SI: v_mul_hi_i32 [[TMP:v[0-9]+]], [[VAL]], [[MAGIC]]
-; SI: v_add_i32
+; SI: v_add_{{[iu]}}32
 ; SI: v_lshrrev_b32
 ; SI: v_ashrrev_i32
-; SI: v_add_i32
+; SI: v_add_{{[iu]}}32
 ; SI: buffer_store_dword
 ; SI: s_endpgm
 define amdgpu_kernel void @slow_sdiv_i32_3435(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
diff --git a/test/CodeGen/AMDGPU/sdivrem64.ll b/test/CodeGen/AMDGPU/sdivrem64.ll
index 5ad0d8efaed3f..d51eededd1c48 100644
--- a/test/CodeGen/AMDGPU/sdivrem64.ll
+++ b/test/CodeGen/AMDGPU/sdivrem64.ll
@@ -36,40 +36,13 @@
 ;EG: BFE_UINT
 ;EG: BFE_UINT
 
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN: s_bfe_u32
-; GCN-NOT: v_mad_f32
-; SI-NOT: v_lshr_b64
-; VI-NOT: v_lshrrev_b64
-; GCN: s_endpgm
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0x4f800000,
+;GCN: v_rcp_f32_e32
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x5f7ffffc
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x2f800000
+;GCN: v_trunc_f32_e32
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0xcf800000
+;GCN: s_endpgm
 define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
   %result = sdiv i64 %x, %y
   store i64 %result, i64 addrspace(1)* %out
@@ -110,39 +83,12 @@ define amdgpu_kernel void @s_test_sdiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ;EG: BFE_UINT
 ;EG: AND_INT {{.*}}, 1,
 
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN-NOT: v_mad_f32
-;SI-NOT: v_lshr_b64
-;VI-NOT: v_lshrrev_b64
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0x4f800000,
+;GCN: v_rcp_f32_e32
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x5f7ffffc
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x2f800000
+;GCN: v_trunc_f32_e32
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0xcf800000
 ;GCN: s_endpgm
 define amdgpu_kernel void @s_test_srem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
   %result = urem i64 %x, %y
diff --git a/test/CodeGen/AMDGPU/sdwa-gfx9.mir b/test/CodeGen/AMDGPU/sdwa-gfx9.mir
index 90cb14bf50d30..2196e7e65c0cf 100644
--- a/test/CodeGen/AMDGPU/sdwa-gfx9.mir
+++ b/test/CodeGen/AMDGPU/sdwa-gfx9.mir
@@ -3,20 +3,20 @@
 # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=si-peephole-sdwa -o - %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 
 # GCN-LABEL: {{^}}name: add_shr_i32
-# GCN: [[SMOV:%[0-9]+]] = S_MOV_B32 123
+# GCN: [[SMOV:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 123
 
-# CI: [[SHIFT:%[0-9]+]] = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
-# CI: %{{[0-9]+}} = V_ADD_I32_e32 [[SMOV]], killed [[SHIFT]], implicit-def %vcc, implicit %exec
+# CI: [[SHIFT:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
+# CI: %{{[0-9]+}}:vgpr_32 = V_ADD_I32_e32 [[SMOV]], killed [[SHIFT]], implicit-def %vcc, implicit %exec
 
-# VI: [[VMOV:%[0-9]+]] = V_MOV_B32_e32 [[SMOV]], implicit %exec
-# VI: %{{[0-9]+}} = V_ADD_I32_sdwa 0, [[VMOV]], 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit-def %vcc, implicit %exec
+# VI: [[VMOV:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[SMOV]], implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_ADD_I32_sdwa 0, [[VMOV]], 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit-def %vcc, implicit %exec
 
-# GFX9: %{{[0-9]+}} = V_ADD_I32_sdwa 0, [[SMOV]], 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit-def %vcc, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_ADD_I32_sdwa 0, [[SMOV]], 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit-def %vcc, implicit %exec
 
 ---
 name:            add_shr_i32
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -33,7 +33,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
@@ -49,18 +49,18 @@ body:             |
 
 # GCN-LABEL: {{^}}name: trunc_shr_f32
 
-# CI: [[SHIFT:%[0-9]+]] = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
-# CI: %{{[0-9]+}} = V_TRUNC_F32_e64 0, killed [[SHIFT]], 1, 2, implicit-def %vcc, implicit %exec
+# CI: [[SHIFT:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
+# CI: %{{[0-9]+}}:vgpr_32 = V_TRUNC_F32_e64 0, killed [[SHIFT]], 1, 2, implicit-def %vcc, implicit %exec
 
-# VI: [[SHIFT:%[0-9]+]] = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
-# VI: %{{[0-9]+}} = V_TRUNC_F32_e64 0, killed [[SHIFT]], 1, 2, implicit-def %vcc, implicit %exec
+# VI: [[SHIFT:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 16, %{{[0-9]+}}, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_TRUNC_F32_e64 0, killed [[SHIFT]], 1, 2, implicit-def %vcc, implicit %exec
 
-#GFX9: %{{[0-9]+}} = V_TRUNC_F32_sdwa 0, %{{[0-9]+}}, 1, 2, 6, 0, 5, implicit %exec
+#GFX9: %{{[0-9]+}}:vgpr_32 = V_TRUNC_F32_sdwa 0, %{{[0-9]+}}, 1, 2, 6, 0, 5, implicit %exec
 
 ---
 name:            trunc_shr_f32
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -76,7 +76,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
diff --git a/test/CodeGen/AMDGPU/sdwa-peephole-instr.mir b/test/CodeGen/AMDGPU/sdwa-peephole-instr.mir
index ff1b2ad73ef0b..0d1534e3f4e84 100644
--- a/test/CodeGen/AMDGPU/sdwa-peephole-instr.mir
+++ b/test/CodeGen/AMDGPU/sdwa-peephole-instr.mir
@@ -3,35 +3,35 @@
 
 # GFX89-LABEL: {{^}}name: vop1_instructions
 
-# GFX89: %{{[0-9]+}} = V_MOV_B32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_FRACT_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
-# GFX89: %{{[0-9]+}} = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_CVT_U32_F32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_FRACT_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_CVT_U32_F32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
 
 
-# GFX89: %{{[0-9]+}} = V_MOV_B32_sdwa 0, %{{[0-9]+}}, 0, 6, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_FRACT_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
-# GFX89: %{{[0-9]+}} = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_CVT_U32_F32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
-# GFX89: %{{[0-9]+}} = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_sdwa 0, %{{[0-9]+}}, 0, 6, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_FRACT_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_CVT_U32_F32_sdwa 0, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, implicit %exec
 
 
-# VI: %{{[0-9]+}} = V_FRACT_F32_sdwa 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
-# VI: %{{[0-9]+}} = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 1, 0, 5, 0, 5, implicit %exec
-# VI: %{{[0-9]+}} = V_CVT_U32_F32_sdwa 1, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
-# VI: %{{[0-9]+}} = V_CVT_F32_I32_e64 %{{[0-9]+}}, 0, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_FRACT_F32_sdwa 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 1, 0, 5, 0, 5, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_CVT_U32_F32_sdwa 1, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_CVT_F32_I32_e64 %{{[0-9]+}}, 0, 1, implicit %exec
 
-# GFX9: %{{[0-9]+}} = V_FRACT_F32_sdwa 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
-# GFX9: %{{[0-9]+}} = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 1, 0, 5, 0, 5, implicit %exec
-# GFX9: %{{[0-9]+}} = V_CVT_U32_F32_sdwa 1, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
-# GFX9: %{{[0-9]+}} = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 1, 5, 0, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_FRACT_F32_sdwa 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_SIN_F32_sdwa 0, %{{[0-9]+}}, 1, 0, 5, 0, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_CVT_U32_F32_sdwa 1, %{{[0-9]+}}, 0, 5, 0, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_CVT_F32_I32_sdwa 0, %{{[0-9]+}}, 0, 1, 5, 0, 5, implicit %exec
 
 
 ---
 name:            vop1_instructions
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -85,7 +85,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
@@ -148,45 +148,45 @@ body:             |
 # GCN-LABEL: {{^}}name: vop2_instructions
 
 
-# VI: %{{[0-9]+}} = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit %exec
-# VI: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 6, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 6, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
 
-# GFX9: %{{[0-9]+}} = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 0, 6, 5, implicit %exec
-# GFX9: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F32_e32 %{{[0-9]+}}, %{{[0-9]+}}, %{{[0-9]+}}, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F16_e32 %{{[0-9]+}}, %{{[0-9]+}}, %{{[0-9]+}}, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_e32 %{{[0-9]+}}, %{{[0-9]+}}, %{{[0-9]+}}, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_e32 %{{[0-9]+}}, %{{[0-9]+}}, %{{[0-9]+}}, implicit %exec
 
 
-# VI: %{{[0-9]+}} = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
-# VI: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 6, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 6, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, %{{[0-9]+}}, 0, 0, 6, 0, 5, 1, implicit %exec
 
-# GFX9: %{{[0-9]+}} = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
-# GFX9: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F32_e64 0, 23, 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F16_e64 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_AND_B32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 5, 0, 6, 5, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_e64 0, 23, 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_e64 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 0, implicit %exec
 
 
-# VI: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_SUB_F16_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F32_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, %{{[0-9]+}}, 1, 0, 6, 0, 6, 1, implicit %exec
-# VI: %{{[0-9]+}} = V_MAC_F16_e64 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 2, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, %{{[0-9]+}}, 1, 0, 6, 0, 6, 1, implicit %exec
+# VI: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_e64 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 2, implicit %exec
 
-# GFX9: %{{[0-9]+}} = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_SUB_F16_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F32_e64 1, 23, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, 0, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MAC_F16_e64 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 2, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_ADD_F32_sdwa 0, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 5, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_SUB_F16_sdwa 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 0, 5, 0, 6, 1, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F32_e64 1, 23, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, 0, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MAC_F16_e64 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 1, %{{[0-9]+}}, 0, 2, implicit %exec
 
 name:            vop2_instructions
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -252,7 +252,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
@@ -324,7 +324,7 @@ body:             |
 
 # GCN-LABEL: {{^}}name: vopc_instructions
 
-# GFX89: %{{[0-9]+}} = V_MOV_B32_e32 123, implicit %exec
+# GFX89: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_e32 123, implicit %exec
 # GFX89: %vcc = V_CMP_EQ_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit %exec
 # GFX89: %vcc = V_CMPX_GT_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit-def %exec, implicit %exec
 # GFX89: %vcc = V_CMP_LT_I32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit %exec
@@ -332,16 +332,16 @@ body:             |
 
 
 # VI: %vcc = V_CMP_EQ_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit %exec
-# VI: %{{[0-9]+}} = V_CMPX_GT_F32_e64 0, 23, 0, killed %{{[0-9]+}}, 0, implicit-def %exec, implicit %exec
+# VI: %{{[0-9]+}}:sreg_64 = V_CMPX_GT_F32_e64 0, 23, 0, killed %{{[0-9]+}}, 0, implicit-def %exec, implicit %exec
 # VI: %vcc = V_CMP_LT_I32_sdwa 0, %{{[0-9]+}}, 0, %3, 0, 6, 4, implicit-def %vcc, implicit %exec
-# VI: %{{[0-9]+}} = V_CMPX_EQ_I32_e64 23, killed %{{[0-9]+}}, implicit-def %exec, implicit %exec
+# VI: %{{[0-9]+}}:sreg_64 = V_CMPX_EQ_I32_e64 23, killed %{{[0-9]+}}, implicit-def %exec, implicit %exec
 
 # GFX9: %vcc = V_CMP_EQ_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MOV_B32_e32 23, implicit %exec
-# GFX9: %{{[0-9]+}} = V_CMPX_GT_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit-def %exec, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_e32 23, implicit %exec
+# GFX9: %{{[0-9]+}}:sreg_64 = V_CMPX_GT_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit-def %exec, implicit %exec
 # GFX9: %vcc = V_CMP_LT_I32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit %exec
-# GFX9: %{{[0-9]+}} = V_MOV_B32_e32 23, implicit %exec
-# GFX9: %{{[0-9]+}} = V_CMPX_EQ_I32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit-def %exec, implicit %exec
+# GFX9: %{{[0-9]+}}:vgpr_32 = V_MOV_B32_e32 23, implicit %exec
+# GFX9: %{{[0-9]+}}:sreg_64 = V_CMPX_EQ_I32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 0, 6, 4, implicit-def %vcc, implicit-def %exec, implicit %exec
 
 
 # VI: %vcc = V_CMP_EQ_F32_sdwa 0, %{{[0-9]+}}, 0, %{{[0-9]+}}, 1, 6, 4, implicit-def %vcc, implicit %exec
@@ -364,7 +364,7 @@ body:             |
 
 name:            vopc_instructions
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -397,7 +397,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
diff --git a/test/CodeGen/AMDGPU/sdwa-peephole.ll b/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 0d181c2c34b85..de5229e0550a0 100644
--- a/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -4,10 +4,11 @@
 
 ; GCN-LABEL: {{^}}add_shr_i32:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
-; NOSDWA-NOT: v_add_i32_sdwa
+; NOSDWA: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
+; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
 
-; SDWA: v_add_i32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9: v_add_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
 
 define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %a = load i32, i32 addrspace(1)* %in, align 4
@@ -19,11 +20,11 @@ define amdgpu_kernel void @add_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)*
 
 ; GCN-LABEL: {{^}}sub_shr_i32:
 ; NOSDWA: v_lshrrev_b32_e32 v[[DST:[0-9]+]], 16, v{{[0-9]+}}
-; NOSDWA: v_subrev_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
-; NOSDWA-NOT: v_subrev_i32_sdwa
-
-; SDWA: v_subrev_i32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; NOSDWA: v_subrev_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v[[DST]]
+; NOSDWA-NOT: v_subrev_{{(_co)?}}_u32_sdwa
 
+; VI: v_subrev_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9: v_sub_u32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 define amdgpu_kernel void @sub_shr_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %a = load i32, i32 addrspace(1)* %in, align 4
   %shr = lshr i32 %a, 16
@@ -426,9 +427,9 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}add_bb_v2i16:
-; NOSDWA-NOT: v_add_i32_sdwa
+; NOSDWA-NOT: v_add_{{(_co)?}}_u32_sdwa
 
-; VI: v_add_i32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; VI: v_add_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
 
 ; GFX9: v_pk_add_u16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 
@@ -496,3 +497,26 @@ entry:
   store <8 x i8> %tmp19, <8 x i8> addrspace(1)* %arrayidx5, align 8
   ret void
 }
+
+; GCN-LABEL: {{^}}sdwa_crash_inlineasm_de
+; GCN: s_mov_b32 s{{[0-9]+}}, 0xffff
+; GCN: v_and_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_or_b32_e32 v{{[0-9]+}}, 0x10000,
+define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb11, %bb
+  %tmp = phi <2 x i32> [ %tmp12, %bb11 ], [ undef, %bb ]
+  br i1 true, label %bb2, label %bb11
+
+bb2:                                              ; preds = %bb1
+  %tmp3 = call i32 asm "v_and_b32_e32 $0, $1, $2", "=v,s,v"(i32 65535, i32 undef) #1
+  %tmp5 = or i32 %tmp3, 65536
+  %tmp6 = insertelement <2 x i32> %tmp, i32 %tmp5, i64 0
+  br label %bb11
+
+bb11:                                             ; preds = %bb10, %bb2
+  %tmp12 = phi <2 x i32> [ %tmp6, %bb2 ], [ %tmp, %bb1 ]
+  br label %bb1
+}
diff --git a/test/CodeGen/AMDGPU/sdwa-preserve.mir b/test/CodeGen/AMDGPU/sdwa-preserve.mir
new file mode 100644
index 0000000000000..99a000cbd39a9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/sdwa-preserve.mir
@@ -0,0 +1,56 @@
+# RUN: llc -march=amdgcn -mcpu=fiji -start-before=si-peephole-sdwa -verify-machineinstrs -o - %s | FileCheck -check-prefix=SDWA %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -start-before=si-peephole-sdwa -verify-machineinstrs -o - %s | FileCheck -check-prefix=SDWA %s
+
+# SDWA-LABEL: {{^}}add_f16_u32_preserve
+
+# SDWA: flat_load_dword [[FIRST:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]
+# SDWA: flat_load_dword [[SECOND:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]
+
+# SDWA: v_mul_f32_sdwa [[RES:v[0-9]+]], [[FIRST]], [[SECOND]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_3
+# SDWA: v_add_f16_sdwa [[RES:v[0-9]+]], [[FIRST]], [[SECOND]] dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:WORD_0 src1_sel:WORD_1
+
+# SDWA: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], [[RES]]
+
+---
+name:            add_f16_u32_preserve
+tracksRegLiveness: true
+registers:       
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vreg_64 }
+  - { id: 2, class: sreg_64 }
+  - { id: 3, class: vgpr_32 }
+  - { id: 4, class: vgpr_32 }
+  - { id: 5, class: vgpr_32 }
+  - { id: 6, class: vgpr_32 }
+  - { id: 7, class: vgpr_32 }
+  - { id: 8, class: vgpr_32 }
+  - { id: 9, class: vgpr_32 }
+  - { id: 10, class: vgpr_32 }
+  - { id: 11, class: vgpr_32 }
+  - { id: 12, class: vgpr_32 }
+  - { id: 13, class: vgpr_32 }
+body:             |
+  bb.0:
+    liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
+  
+    %2 = COPY %sgpr30_sgpr31
+    %1 = COPY %vgpr2_vgpr3
+    %0 = COPY %vgpr0_vgpr1
+    %3 = FLAT_LOAD_DWORD %0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+    %4 = FLAT_LOAD_DWORD %1, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4)
+
+    %5 = V_AND_B32_e32 65535, %3, implicit %exec
+    %6 = V_LSHRREV_B32_e64 16, %4, implicit %exec
+    %7 = V_BFE_U32 %3, 8, 8, implicit %exec
+    %8 = V_LSHRREV_B32_e32 24, %4, implicit %exec
+
+    %9 = V_ADD_F16_e64 0, %5, 0, %6, 0, 0, implicit %exec
+    %10 = V_LSHLREV_B16_e64 8, %9, implicit %exec
+    %11 = V_MUL_F32_e64 0, %7, 0, %8, 0, 0, implicit %exec
+    %12 = V_LSHLREV_B32_e64 16, %11, implicit %exec
+
+    %13 = V_OR_B32_e64 %10, %12, implicit %exec
+
+    FLAT_STORE_DWORD %0, %13, 0, 0, 0, implicit %exec, implicit %flat_scr :: (store 4)
+    %sgpr30_sgpr31 = COPY %2
+    S_SETPC_B64_return %sgpr30_sgpr31
diff --git a/test/CodeGen/AMDGPU/sdwa-vop2-64bit.mir b/test/CodeGen/AMDGPU/sdwa-vop2-64bit.mir
index bd222adf6a68c..c50601e79f2e1 100644
--- a/test/CodeGen/AMDGPU/sdwa-vop2-64bit.mir
+++ b/test/CodeGen/AMDGPU/sdwa-vop2-64bit.mir
@@ -6,15 +6,15 @@
 
 # GCN-LABEL: {{^}}name: vop2_64bit
 
-# GCN: %{{[0-9]+}} = V_BCNT_U32_B32_e64 %{{[0-9]+}}, killed %{{[0-9]+}}, implicit-def %vcc, implicit %exec
-# GCN: %{{[0-9]+}} = V_BFM_B32_e64 %{{[0-9]+}}, killed %{{[0-9]+}}, implicit-def %vcc, implicit %exec
-# GCN: %{{[0-9]+}} = V_CVT_PKNORM_I16_F32_e64 0, %{{[0-9]+}}, 0, killed %{{[0-9]+}}, 0, implicit-def %vcc, implicit %exec
-# GCN: %{{[0-9]+}} = V_READLANE_B32 killed %{{[0-9]+}}, 0, implicit-def %vcc, implicit %exec
+# GCN: %{{[0-9]+}}:vgpr_32 = V_BCNT_U32_B32_e64 %{{[0-9]+}}, killed %{{[0-9]+}}, implicit-def %vcc, implicit %exec
+# GCN: %{{[0-9]+}}:vgpr_32 = V_BFM_B32_e64 %{{[0-9]+}}, killed %{{[0-9]+}}, implicit-def %vcc, implicit %exec
+# GCN: %{{[0-9]+}}:vgpr_32 = V_CVT_PKNORM_I16_F32_e64 0, %{{[0-9]+}}, 0, killed %{{[0-9]+}}, 0, implicit-def %vcc, implicit %exec
+# GCN: %{{[0-9]+}}:sgpr_32 = V_READLANE_B32 killed %{{[0-9]+}}, 0, implicit-def %vcc, implicit %exec
 
 ---
 name:            vop2_64bit
 tracksRegLiveness: true
-registers:       
+registers:
   - { id: 0, class: vreg_64 }
   - { id: 1, class: vreg_64 }
   - { id: 2, class: sreg_64 }
@@ -37,7 +37,7 @@ registers:
 body:             |
   bb.0:
     liveins: %vgpr0_vgpr1, %vgpr2_vgpr3, %sgpr30_sgpr31
-  
+
     %2 = COPY %sgpr30_sgpr31
     %1 = COPY %vgpr2_vgpr3
     %0 = COPY %vgpr0_vgpr1
diff --git a/test/CodeGen/AMDGPU/select-vectors.ll b/test/CodeGen/AMDGPU/select-vectors.ll
index b77ebcf5bf529..e5c8191b24b99 100644
--- a/test/CodeGen/AMDGPU/select-vectors.ll
+++ b/test/CodeGen/AMDGPU/select-vectors.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -verify-machineinstrs -march=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -verify-machineinstrs -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -verify-machineinstrs -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -march=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false -verify-machineinstrs -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; Test expansion of scalar selects on vectors.
 ; Evergreen not enabled since it seems to be having problems with doubles.
@@ -215,9 +215,9 @@ define amdgpu_kernel void @select_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32>
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[ALO]]
 ; GCN-DAG: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}}
 
-; GCN: v_cndmask_b32_e32
-; GCN: v_mov_b32_e32 v{{[0-9]+}}, s[[BLO]]
-; GCN: v_cndmask_b32_e32
+; GCN-DAG: v_cndmask_b32_e32
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[BLO]]
+; GCN-DAG: v_cndmask_b32_e32
 ; GCN: buffer_store_dwordx2
 define amdgpu_kernel void @s_select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b, i32 %c) #0 {
   %cmp = icmp eq i32 %c, 0
diff --git a/test/CodeGen/AMDGPU/select.f16.ll b/test/CodeGen/AMDGPU/select.f16.ll
index e79ce3af0cf9d..9030baa04c584 100644
--- a/test/CodeGen/AMDGPU/select.f16.ll
+++ b/test/CodeGen/AMDGPU/select.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}select_f16:
 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
@@ -8,9 +8,9 @@
 ; GCN: buffer_load_ushort v[[D_F16:[0-9]+]]
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
 ; SI:  v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
-; SI:  v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]]
-; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
+; SI-DAG:  v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
 ; SI:  v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
 ; VI:  v_cmp_lt_f16_e32 vcc, v[[A_F16]], v[[B_F16]]
@@ -39,9 +39,9 @@ entry:
 ; GCN: buffer_load_ushort v[[C_F16:[0-9]+]]
 ; GCN: buffer_load_ushort v[[D_F16:[0-9]+]]
 ; SI:  v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
-; SI:  v_cmp_lt_f32_e32 vcc, 0.5, v[[B_F32]]
-; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
+; SI-DAG:  v_cmp_lt_f32_e32 vcc, 0.5, v[[B_F32]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
 ; SI:  v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
 ; VI:  v_cmp_lt_f16_e32 vcc, 0.5, v[[B_F16]]
@@ -68,9 +68,9 @@ entry:
 ; GCN: buffer_load_ushort v[[C_F16:[0-9]+]]
 ; GCN: buffer_load_ushort v[[D_F16:[0-9]+]]
 ; SI:  v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
-; SI:  v_cmp_gt_f32_e32 vcc, 0.5, v[[A_F32]]
-; SI:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
-; SI:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
+; SI-DAG:  v_cmp_gt_f32_e32 vcc, 0.5, v[[A_F32]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]]
+; SI-DAG:  v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]]
 ; SI:  v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]]
 ; SI:  v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
 
diff --git a/test/CodeGen/AMDGPU/select64.ll b/test/CodeGen/AMDGPU/select64.ll
index 3b4c925a87a09..0a48227f5c500 100644
--- a/test/CodeGen/AMDGPU/select64.ll
+++ b/test/CodeGen/AMDGPU/select64.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -march=amdgcn -mcpu=tahiti -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s
 
 ; CHECK-LABEL: {{^}}select0:
diff --git a/test/CodeGen/AMDGPU/selectcc.ll b/test/CodeGen/AMDGPU/selectcc.ll
index 7eca22913987e..73e4368c52ba1 100644
--- a/test/CodeGen/AMDGPU/selectcc.ll
+++ b/test/CodeGen/AMDGPU/selectcc.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 ; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}selectcc_i64:
diff --git a/test/CodeGen/AMDGPU/sendmsg-m0-hazard.mir b/test/CodeGen/AMDGPU/sendmsg-m0-hazard.mir
new file mode 100644
index 0000000000000..5dfd5aa384fd6
--- /dev/null
+++ b/test/CodeGen/AMDGPU/sendmsg-m0-hazard.mir
@@ -0,0 +1,49 @@
+# RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,VI %s
+# RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,CI %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -run-pass  post-RA-hazard-rec %s -o - | FileCheck -check-prefixes=GCN,SI %s
+
+---
+name: m0_sendmsg
+body: |
+  ; GCN-LABEL: name: m0_sendmsg
+  ; GCN:  %m0 = S_MOV_B32 -1
+  ; VI-NEXT: S_NOP 0
+  ; GFX9-NEXT: S_NOP 0
+  ; GCN-NEXT: S_SENDMSG 3, implicit %exec, implicit %m0
+
+  bb.0:
+    %m0 = S_MOV_B32 -1
+    S_SENDMSG 3, implicit %exec, implicit %m0
+    S_ENDPGM
+...
+---
+
+name: m0_sendmsghalt
+body: |
+  ; GCN-LABEL: name: m0_sendmsghalt
+  ; GCN:  %m0 = S_MOV_B32 -1
+  ; VI-NEXT: S_NOP 0
+  ; GFX9-NEXT: S_NOP 0
+  ; GCN-NEXT: S_SENDMSGHALT 3, implicit %exec, implicit %m0
+
+  bb.0:
+    %m0 = S_MOV_B32 -1
+    S_SENDMSGHALT 3, implicit %exec, implicit %m0
+    S_ENDPGM
+...
+---
+
+name: m0_ttracedata
+body: |
+  ; GCN-LABEL: name: m0_ttracedata
+  ; GCN:  %m0 = S_MOV_B32 -1
+  ; VI-NEXT: S_NOP 0
+  ; GFX9-NEXT: S_NOP 0
+  ; GCN-NEXT: S_TTRACEDATA implicit %m0
+
+  bb.0:
+    %m0 = S_MOV_B32 -1
+    S_TTRACEDATA implicit %m0
+    S_ENDPGM
+...
diff --git a/test/CodeGen/AMDGPU/setcc-fneg-constant.ll b/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
index bcaa1aa54c15f..22827b2cc2bb9 100644
--- a/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
+++ b/test/CodeGen/AMDGPU/setcc-fneg-constant.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
 
 ; Test fcmp pred (fneg x), c -> fcmp (swapped pred) x, -c combine.
 
diff --git a/test/CodeGen/AMDGPU/setcc.ll b/test/CodeGen/AMDGPU/setcc.ll
index a3bf167e756af..442a3163270e9 100644
--- a/test/CodeGen/AMDGPU/setcc.ll
+++ b/test/CodeGen/AMDGPU/setcc.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
@@ -416,4 +416,56 @@ bb2:
   ret void
 }
 
+; FUNC-LABEL: setcc_v2i32_expand
+; GCN: v_cmp_gt_i32
+; GCN: v_cmp_gt_i32
+define amdgpu_kernel void @setcc_v2i32_expand(
+  <2 x i32> addrspace(1)* %a,
+  <2 x i32> addrspace(1)* %b,
+  <2 x i32> addrspace(1)* %c,
+  <2 x float> addrspace(1)* %r) {
+entry:
+  %a.val = load <2 x i32>, <2 x i32> addrspace(1)* %a
+  %b.val = load <2 x i32>, <2 x i32> addrspace(1)* %b
+  %c.val = load <2 x i32>, <2 x i32> addrspace(1)* %c
+
+  %icmp.val.1 = icmp sgt <2 x i32> %a.val, <i32 1, i32 1>
+  %zext.val.1 = zext <2 x i1> %icmp.val.1 to <2 x i32>
+  %shl.val.1 = shl nuw <2 x i32> %zext.val.1, <i32 31, i32 31>
+  %xor.val.1 = xor <2 x i32> %shl.val.1, %b.val
+  %bitcast.val.1 = bitcast <2 x i32> %xor.val.1 to <2 x float>
+  %icmp.val.2 = icmp sgt <2 x i32> %c.val, <i32 1199570944, i32 1199570944>
+  %select.val.1 = select <2 x i1> %icmp.val.2, <2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> %bitcast.val.1
+
+  store <2 x float> %select.val.1, <2 x float> addrspace(1)* %r
+  ret void
+}
+
+; FUNC-LABEL: setcc_v4i32_expand
+; GCN: v_cmp_gt_i32
+; GCN: v_cmp_gt_i32
+; GCN: v_cmp_gt_i32
+; GCN: v_cmp_gt_i32
+define amdgpu_kernel void @setcc_v4i32_expand(
+  <4 x i32> addrspace(1)* %a,
+  <4 x i32> addrspace(1)* %b,
+  <4 x i32> addrspace(1)* %c,
+  <4 x float> addrspace(1)* %r) {
+entry:
+  %a.val = load <4 x i32>, <4 x i32> addrspace(1)* %a
+  %b.val = load <4 x i32>, <4 x i32> addrspace(1)* %b
+  %c.val = load <4 x i32>, <4 x i32> addrspace(1)* %c
+
+  %icmp.val.1 = icmp sgt <4 x i32> %a.val, <i32 1, i32 1, i32 1, i32 1>
+  %zext.val.1 = zext <4 x i1> %icmp.val.1 to <4 x i32>
+  %shl.val.1 = shl nuw <4 x i32> %zext.val.1, <i32 31, i32 31, i32 31, i32 31>
+  %xor.val.1 = xor <4 x i32> %shl.val.1, %b.val
+  %bitcast.val.1 = bitcast <4 x i32> %xor.val.1 to <4 x float>
+  %icmp.val.2 = icmp sgt <4 x i32> %c.val, <i32 1199570944, i32 1199570944, i32 1199570944, i32 1199570944>
+  %select.val.1 = select <4 x i1> %icmp.val.2, <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <4 x float> %bitcast.val.1
+
+  store <4 x float> %select.val.1, <4 x float> addrspace(1)* %r
+  ret void
+}
+
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/sext-in-reg.ll b/test/CodeGen/AMDGPU/sext-in-reg.ll
index 5b4d9ed259b60..4cf284630c223 100644
--- a/test/CodeGen/AMDGPU/sext-in-reg.ll
+++ b/test/CodeGen/AMDGPU/sext-in-reg.ll
@@ -1,7 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=GFX89 -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GFX9 -check-prefix=GFX89 -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600---amdgiz -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
 
 ; FIXME: i16 promotion pass ruins the scalar cases when legal.
 ; FIXME: r600 fails verifier
@@ -152,14 +152,14 @@ define amdgpu_kernel void @sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 %a
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
 ; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1
 ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 
 ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -179,14 +179,14 @@ define amdgpu_kernel void @v_sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 a
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
 ; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 8
 ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 
 ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -206,14 +206,14 @@ define amdgpu_kernel void @v_sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 a
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
 ; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 16
 ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 
 ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -233,11 +233,11 @@ define amdgpu_kernel void @v_sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}},
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}},
 
 ; GCN: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[SHR]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[SHR]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -463,7 +463,7 @@ define amdgpu_kernel void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocaptu
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}
 
 ; GCN-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1
@@ -471,7 +471,7 @@ define amdgpu_kernel void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocaptu
 ; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]]
 ; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[HI]]
 ; SI: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -493,7 +493,7 @@ define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %o
 ; SI: buffer_load_dwordx2
 ; SI: v_lshl_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}},
 
-; GFX89: flat_load_dwordx2
+; GFX89: {{flat|global}}_load_dwordx2
 ; GFX89: v_lshlrev_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}},
 
 ; GCN-DAG: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]]
@@ -501,7 +501,7 @@ define amdgpu_kernel void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %o
 ; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[SHR]]
 
 ; SI: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
-; GFX89: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
+; GFX89: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}}
 define amdgpu_kernel void @v_sext_in_reg_i32_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
   %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid
@@ -557,7 +557,7 @@ define amdgpu_kernel void @s_sext_in_reg_i2_i16(i16 addrspace(1)* %out, i32 addr
 }
 
 ; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16:
-; GCN: {{buffer|flat}}_load_ushort [[VAL:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[VAL:v[0-9]+]]
 ; GCN: v_bfe_i32 [[BFE:v[0-9]+]], [[VAL]], 0, 1{{$}}
 
 ; GCN: ds_write_b16 v{{[0-9]+}}, [[BFE]]
@@ -574,8 +574,8 @@ define amdgpu_kernel void @v_sext_in_reg_i1_i16(i16 addrspace(3)* %out, i16 addr
 }
 
 ; FUNC-LABEL: {{^}}v_sext_in_reg_i1_i16_nonload:
-; GCN: {{buffer|flat}}_load_ushort [[VAL0:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_ushort [[VAL1:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[VAL0:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_ushort [[VAL1:v[0-9]+]]
 
 ; SI: v_lshlrev_b32_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]
 ; GFX89: v_lshlrev_b16_e32 [[REG:v[0-9]+]], [[VAL1]], [[VAL0]]
@@ -620,7 +620,7 @@ define amdgpu_kernel void @s_sext_in_reg_i2_i16_arg(i16 addrspace(1)* %out, i16
 
 ; SI: s_sext_i32_i8 [[SSEXT:s[0-9]+]], [[VAL]]
 ; SI: v_mov_b32_e32 [[VSEXT:v[0-9]+]], [[SSEXT]]
-; SI: buffer_store_short [[VBFE]]
+; SI: buffer_store_short [[VSEXT]]
 
 ; GFX89: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 8{{$}}
 ; GFX89: s_sext_i32_i16 s{{[0-9]+}}, s{{[0-9]+}}
@@ -663,10 +663,10 @@ define amdgpu_kernel void @sext_in_reg_v2i1_to_v2i16(<2 x i16> addrspace(1)* %ou
 
 ; FUNC-LABEL: {{^}}sext_in_reg_v3i1_to_v3i16:
 ; GFX9: v_pk_add_u16
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}
+; GFX9: v_pk_add_u16
+; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 15, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 15, v{{[0-9]+}}
 define amdgpu_kernel void @sext_in_reg_v3i1_to_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %a, <3 x i16> %b) #0 {
   %c = add <3 x i16> %a, %b ; add to prevent folding into extload
@@ -702,10 +702,11 @@ define amdgpu_kernel void @sext_in_reg_v2i8_to_v2i16(<2 x i16> addrspace(1)* %ou
 
 ; FUNC-LABEL: {{^}}sext_in_reg_v3i8_to_v3i16:
 ; GFX9: v_pk_add_u16
-; GFX9: v_pk_add_u16
-; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
+
+; GFX9: v_pk_add_u16
+; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 define amdgpu_kernel void @sext_in_reg_v3i8_to_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> %a, <3 x i16> %b) #0 {
   %c = add <3 x i16> %a, %b ; add to prevent folding into extload
diff --git a/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/test/CodeGen/AMDGPU/sgpr-control-flow.ll
index 8e18ab5554e45..575938b5a5cb2 100644
--- a/test/CodeGen/AMDGPU/sgpr-control-flow.ll
+++ b/test/CodeGen/AMDGPU/sgpr-control-flow.ll
@@ -37,7 +37,7 @@ endif:
 ; SI: s_cmp_lg_u32
 ; SI: s_cbranch_scc0 [[IF:BB[0-9]+_[0-9]+]]
 
-; SI: ; BB#1: ; %else
+; SI: ; %bb.1: ; %else
 ; SI: s_load_dword [[LOAD0:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xe
 ; SI: s_load_dword [[LOAD1:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xf
 ; SI-NOT: add
diff --git a/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll b/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
index 8250bad7b0a10..ed11c2c3e22f5 100644
--- a/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
+++ b/test/CodeGen/AMDGPU/sgpr-copy-duplicate-operand.ll
@@ -5,7 +5,7 @@
 ; used in an REG_SEQUENCE that also needs to be handled.
 
 ; SI-LABEL: {{^}}test_dup_operands:
-; SI: v_add_i32_e32
+; SI: v_add_{{[iu]}}32_e32
 define amdgpu_kernel void @test_dup_operands(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrspace(1)* noalias %in) {
   %a = load <2 x i32>, <2 x i32> addrspace(1)* %in
   %lo = extractelement <2 x i32> %a, i32 0
diff --git a/test/CodeGen/AMDGPU/sgpr-copy.ll b/test/CodeGen/AMDGPU/sgpr-copy.ll
index 3b24cf82d783b..9a431bc387e08 100644
--- a/test/CodeGen/AMDGPU/sgpr-copy.ll
+++ b/test/CodeGen/AMDGPU/sgpr-copy.ll
@@ -321,7 +321,7 @@ ENDIF69:                                          ; preds = %LOOP68
 ; CHECK: s_cmp_eq_u32
 ; CHECK: s_cbranch_scc0 [[END:BB[0-9]+_[0-9]+]]
 
-; CHECK: v_add_i32_e32 v[[ADD:[0-9]+]], vcc, 1, v{{[0-9]+}}
+; CHECK: v_add_{{[iu]}}32_e32 v[[ADD:[0-9]+]], vcc, 1, v{{[0-9]+}}
 
 ; [[END]]:
 ; CHECK: image_sample v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+}}:[[ADD]]{{\]}}
diff --git a/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll b/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll
index 2f9eed457ab6d..72767d22efc43 100644
--- a/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll
+++ b/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; Extract the high bit of the 1st quarter
 ; GCN-LABEL: {{^}}v_uextract_bit_31_i128:
diff --git a/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll b/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll
index 670287ba79373..26e6a3a52ea61 100644
--- a/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll
+++ b/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll
@@ -1,4 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; FIXME: Fails with -enable-var-scope
 
 ; Make sure 64-bit BFE pattern does a 32-bit BFE on the relevant half.
 
@@ -91,7 +92,7 @@ define amdgpu_kernel void @v_uextract_bit_32_i64(i64 addrspace(1)* %out, i64 add
 ; GCN: buffer_load_dword [[VAL:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; GCN-DAG: v_bfe_u32 v[[BFE:[0-9]+]], [[VAL]], 1, 1{{$}}
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO1:[0-9]+]], v[[ZERO]]
-; GCN: buffer_store_dwordx2 v{{\[}}[[SHIFT]]:[[ZERO1]]{{\]}}
+; GCN: buffer_store_dwordx2 v{{\[}}[[BFE]]:[[ZERO1]]{{\]}}
 define amdgpu_kernel void @v_uextract_bit_33_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) #1 {
   %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %id.x
@@ -210,7 +211,7 @@ define amdgpu_kernel void @v_uextract_bit_30_60_i64(i64 addrspace(1)* %out, i64
 ; GCN: buffer_load_dword [[VAL:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}}
 ; GCN-DAG: v_bfe_u32 v[[BFE:[0-9]+]], [[VAL]], 1, 30
 ; GCN-DAG: v_mov_b32_e32 v[[ZERO1:[0-9]+]], v[[ZERO]]
-; GCN: buffer_store_dwordx2 v{{\[}}[[SHIFT]]:[[ZERO1]]{{\]}}
+; GCN: buffer_store_dwordx2 v{{\[}}[[BFE]]:[[ZERO1]]{{\]}}
 define amdgpu_kernel void @v_uextract_bit_33_63_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) #1 {
   %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
   %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %id.x
diff --git a/test/CodeGen/AMDGPU/shl-add-to-add-shl.ll b/test/CodeGen/AMDGPU/shl-add-to-add-shl.ll
index 1cdfec9fdb597..2fe064e5dc953 100644
--- a/test/CodeGen/AMDGPU/shl-add-to-add-shl.ll
+++ b/test/CodeGen/AMDGPU/shl-add-to-add-shl.ll
@@ -5,9 +5,9 @@
 
 ; CHECK-LABEL: {{^}}add_const_offset:
 ; CHECK: v_lshlrev_b32_e32 v[[SHL:[0-9]+]], 4, v0
-; CHECK: v_add_i32_e32 v[[ADD:[0-9]+]], vcc, 0xc80, v[[SHL]]
+; CHECK: v_add_u32_e32 v[[ADD:[0-9]+]], vcc, 0xc80, v[[SHL]]
 ; CHECK-NOT: v_lshl
-; CHECK: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADD]]
+; CHECK: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[ADD]]
 ; CHECK: load_dword v{{[0-9]+}}, v{{\[}}[[ADDRLO]]:
 define amdgpu_kernel void @add_const_offset(i32 addrspace(1)* nocapture %arg) {
 bb:
@@ -24,7 +24,7 @@ bb:
 ; CHECK: v_lshlrev_b32_e32 v[[SHL:[0-9]+]], 4, v0
 ; CHECK: v_or_b32_e32 v[[OR:[0-9]+]], 0x1000, v[[SHL]]
 ; CHECK-NOT: v_lshl
-; CHECK: v_add_i32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[OR]]
+; CHECK: v_add_u32_e32 v[[ADDRLO:[0-9]+]], vcc, s{{[0-9]+}}, v[[OR]]
 ; CHECK: load_dword v{{[0-9]+}}, v{{\[}}[[ADDRLO]]:
 define amdgpu_kernel void @or_const_offset(i32 addrspace(1)* nocapture %arg) {
 bb:
diff --git a/test/CodeGen/AMDGPU/shl.ll b/test/CodeGen/AMDGPU/shl.ll
index 13ac9140b8273..14f4646429426 100644
--- a/test/CodeGen/AMDGPU/shl.ll
+++ b/test/CodeGen/AMDGPU/shl.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; XUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; XUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() #0
 
@@ -476,4 +476,28 @@ define amdgpu_kernel void @test_mul2(i32 %p) {
    ret void
 }
 
+; FUNC-LABEL: {{^}}shl_or_k:
+; SI: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, v{{[0-9]+}}
+; SI: v_or_b32_e32 [[OR:v[0-9]+]], 4, [[SHL]]
+; SI: buffer_store_dword [[OR]]
+define void @shl_or_k(i32 addrspace(1)* %out, i32 %in) {
+  %tmp0 = or i32 %in, 1
+  %tmp2 = shl i32 %tmp0, 2
+  store i32 %tmp2, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}shl_or_k_two_uses:
+; SI: v_or_b32_e32 [[OR:v[0-9]+]], 1, v{{[0-9]+}}
+; SI: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 2, [[OR]]
+; SI-DAG: buffer_store_dword [[OR]]
+; SI-DAG: buffer_store_dword [[SHL]]
+define void @shl_or_k_two_uses(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 %in) {
+  %tmp0 = or i32 %in, 1
+  %tmp2 = shl i32 %tmp0, 2
+  store i32 %tmp2, i32 addrspace(1)* %out0
+  store i32 %tmp0, i32 addrspace(1)* %out1
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/shl.v2i16.ll b/test/CodeGen/AMDGPU/shl.v2i16.ll
index 839854fd575bd..157808b39651b 100644
--- a/test/CodeGen/AMDGPU/shl.v2i16.ll
+++ b/test/CodeGen/AMDGPU/shl.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=CIVI %s
 ; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=CIVI %s
 
@@ -12,10 +12,10 @@
 ; VI: v_lshlrev_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
-; CI: v_lshlrev_b32_e32
-; CI: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CI: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
+; CI-DAG: v_lshlrev_b32_e32
+; CI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}}
+; CI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; CI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
 ; CI: v_or_b32_e32
 define amdgpu_kernel void @s_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
   %result = shl <2 x i16> %lhs, %rhs
@@ -24,8 +24,8 @@ define amdgpu_kernel void @s_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %
 }
 
 ; GCN-LABEL: {{^}}v_shl_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 
 ; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
@@ -55,7 +55,7 @@ define amdgpu_kernel void @v_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> a
 
 ; GCN-LABEL: {{^}}shl_v_s_v2i16:
 ; GFX9: s_load_dword [[RHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @shl_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -70,7 +70,7 @@ define amdgpu_kernel void @shl_v_s_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 
 ; GCN-LABEL: {{^}}shl_s_v_v2i16:
 ; GFX9: s_load_dword [[LHS:s[0-9]+]]
-; GFX9: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GFX9: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]]
 define amdgpu_kernel void @shl_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in, <2 x i16> %sgpr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -84,7 +84,7 @@ define amdgpu_kernel void @shl_s_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16>
 }
 
 ; GCN-LABEL: {{^}}shl_imm_v_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[RHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]]
 ; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], [[RHS]], 8
 define amdgpu_kernel void @shl_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -98,7 +98,7 @@ define amdgpu_kernel void @shl_imm_v_v2i16(<2 x i16> addrspace(1)* %out, <2 x i1
 }
 
 ; GCN-LABEL: {{^}}shl_v_imm_v2i16:
-; GCN: {{buffer|flat}}_load_dword [[LHS:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]]
 ; GFX9: v_pk_lshlrev_b16 [[RESULT:v[0-9]+]], 8, [[LHS]]
 define amdgpu_kernel void @shl_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -112,11 +112,11 @@ define amdgpu_kernel void @shl_v_imm_v2i16(<2 x i16> addrspace(1)* %out, <2 x i1
 }
 
 ; GCN-LABEL: {{^}}v_shl_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @v_shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -131,10 +131,10 @@ define amdgpu_kernel void @v_shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> a
 }
 
 ; GCN-LABEL: {{^}}shl_v_imm_v4i16:
-; GCN: {{buffer|flat}}_load_dwordx2
+; GCN: {{buffer|flat|global}}_load_dwordx2
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
 ; GFX9: v_pk_lshlrev_b16 v{{[0-9]+}}, 8, v{{[0-9]+}}
-; GCN: {{buffer|flat}}_store_dwordx2
+; GCN: {{buffer|flat|global}}_store_dwordx2
 define amdgpu_kernel void @shl_v_imm_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/shl_add_constant.ll b/test/CodeGen/AMDGPU/shl_add_constant.ll
index 9da4bc0280163..26cdae3cbb049 100644
--- a/test/CodeGen/AMDGPU/shl_add_constant.ll
+++ b/test/CodeGen/AMDGPU/shl_add_constant.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
@@ -56,7 +56,7 @@ define amdgpu_kernel void @shl_2_add_999_i32(i32 addrspace(1)* %out, i32 addrspa
 ; FUNC-LABEL: {{^}}test_add_shl_add_constant:
 ; SI-DAG: s_load_dword [[X:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
 ; SI-DAG: s_load_dword [[Y:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xc
-; SI: s_lshl_b32 [[SHL3:s[0-9]+]], [[X]], 3
+; SI-DAG: s_lshl_b32 [[SHL3:s[0-9]+]], [[X]], 3
 ; SI: s_add_i32 [[RESULT:s[0-9]+]], [[SHL3]], [[Y]]
 ; SI: s_addk_i32 [[RESULT]], 0x3d8
 ; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[RESULT]]
diff --git a/test/CodeGen/AMDGPU/shl_add_ptr.ll b/test/CodeGen/AMDGPU/shl_add_ptr.ll
index 9147eb58c6ad2..3e77d876da3ce 100644
--- a/test/CodeGen/AMDGPU/shl_add_ptr.ll
+++ b/test/CodeGen/AMDGPU/shl_add_ptr.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs -mattr=+load-store-opt -enable-misched < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; Test that doing a shift of a pointer with a constant add will be
 ; folded into the constant offset addressing mode even if the add has
@@ -15,10 +15,10 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
 
 ; Make sure the (add tid, 2) << 2 gets folded into the ds's offset as (tid << 2) + 8
 
-; SI-LABEL: {{^}}load_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_read_b32 {{v[0-9]+}}, [[PTR]] offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}load_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_read_b32 {{v[0-9]+}}, [[PTR]] offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @load_shl_base_lds_0(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -32,13 +32,13 @@ define amdgpu_kernel void @load_shl_base_lds_0(float addrspace(1)* %out, i32 add
 ; Make sure once the first use is folded into the addressing mode, the
 ; remaining add use goes through the normal shl + add constant fold.
 
-; SI-LABEL: {{^}}load_shl_base_lds_1:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_read_b32 [[RESULT:v[0-9]+]], [[PTR]] offset:8
-; SI: v_add_i32_e32 [[ADDUSE:v[0-9]+]], vcc, 8, v{{[0-9]+}}
-; SI-DAG: buffer_store_dword [[RESULT]]
-; SI-DAG: buffer_store_dword [[ADDUSE]]
-; SI: s_endpgm
+; GCN-LABEL: {{^}}load_shl_base_lds_1:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_read_b32 [[RESULT:v[0-9]+]], [[PTR]] offset:8
+; GCN: v_add_{{[iu]}}32_e32 [[ADDUSE:v[0-9]+]], vcc, 8, v{{[0-9]+}}
+; GCN-DAG: buffer_store_dword [[RESULT]]
+; GCN-DAG: buffer_store_dword [[ADDUSE]]
+; GCN: s_endpgm
 define amdgpu_kernel void @load_shl_base_lds_1(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -52,9 +52,9 @@ define amdgpu_kernel void @load_shl_base_lds_1(float addrspace(1)* %out, i32 add
 
 @maxlds = addrspace(3) global [65536 x i8] undef, align 4
 
-; SI-LABEL: {{^}}load_shl_base_lds_max_offset
-; SI: ds_read_u8 v{{[0-9]+}}, v{{[0-9]+}} offset:65535
-; SI: s_endpgm
+; GCN-LABEL: {{^}}load_shl_base_lds_max_offset
+; GCN: ds_read_u8 v{{[0-9]+}}, v{{[0-9]+}} offset:65535
+; GCN: s_endpgm
 define amdgpu_kernel void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i8 addrspace(3)* %lds, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 65535
@@ -68,11 +68,11 @@ define amdgpu_kernel void @load_shl_base_lds_max_offset(i8 addrspace(1)* %out, i
 ; The two globals are placed adjacent in memory, so the same base
 ; pointer can be used with an offset into the second one.
 
-; SI-LABEL: {{^}}load_shl_base_lds_2:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: s_mov_b32 m0, -1
-; SI-NEXT: ds_read2st64_b32 {{v\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:1 offset1:9
-; SI: s_endpgm
+; GCN-LABEL: {{^}}load_shl_base_lds_2:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: s_mov_b32 m0, -1
+; GCN-NEXT: ds_read2st64_b32 {{v\[[0-9]+:[0-9]+\]}}, [[PTR]] offset0:1 offset1:9
+; GCN: s_endpgm
 define amdgpu_kernel void @load_shl_base_lds_2(float addrspace(1)* %out) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 64
@@ -85,10 +85,10 @@ define amdgpu_kernel void @load_shl_base_lds_2(float addrspace(1)* %out) #0 {
   ret void
 }
 
-; SI-LABEL: {{^}}store_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_write_b32 [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}store_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_write_b32 [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @store_shl_base_lds_0(float addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -115,10 +115,10 @@ define amdgpu_kernel void @store_shl_base_lds_0(float addrspace(1)* %out, i32 ad
 ; }
 
 
-; SI-LABEL: {{^}}atomic_cmpxchg_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_cmpst_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}}, {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_cmpxchg_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_cmpst_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}}, {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_cmpxchg_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use, i32 %swap) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -130,10 +130,10 @@ define amdgpu_kernel void @atomic_cmpxchg_shl_base_lds_0(i32 addrspace(1)* %out,
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_swap_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_wrxchg_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_swap_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_wrxchg_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_swap_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -144,10 +144,10 @@ define amdgpu_kernel void @atomic_swap_shl_base_lds_0(i32 addrspace(1)* %out, i3
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_add_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_add_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_add_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_add_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_add_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -158,10 +158,10 @@ define amdgpu_kernel void @atomic_add_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_sub_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_sub_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_sub_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_sub_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_sub_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -172,10 +172,10 @@ define amdgpu_kernel void @atomic_sub_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_and_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_and_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_and_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_and_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_and_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -186,10 +186,10 @@ define amdgpu_kernel void @atomic_and_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_or_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_or_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_or_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_or_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_or_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -200,10 +200,10 @@ define amdgpu_kernel void @atomic_or_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_xor_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_xor_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_xor_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_xor_rtn_b32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_xor_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -224,10 +224,10 @@ define amdgpu_kernel void @atomic_xor_shl_base_lds_0(i32 addrspace(1)* %out, i32
 ;   ret void
 ; }
 
-; SI-LABEL: {{^}}atomic_min_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_min_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_min_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_min_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_min_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -238,10 +238,10 @@ define amdgpu_kernel void @atomic_min_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_max_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_max_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_max_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_max_rtn_i32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_max_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -252,10 +252,10 @@ define amdgpu_kernel void @atomic_max_shl_base_lds_0(i32 addrspace(1)* %out, i32
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_umin_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_min_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_umin_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_min_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_umin_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -266,10 +266,10 @@ define amdgpu_kernel void @atomic_umin_shl_base_lds_0(i32 addrspace(1)* %out, i3
   ret void
 }
 
-; SI-LABEL: {{^}}atomic_umax_shl_base_lds_0:
-; SI: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
-; SI: ds_max_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
-; SI: s_endpgm
+; GCN-LABEL: {{^}}atomic_umax_shl_base_lds_0:
+; GCN: v_lshlrev_b32_e32 [[PTR:v[0-9]+]], 2, {{v[0-9]+}}
+; GCN: ds_max_rtn_u32 {{v[0-9]+}}, [[PTR]], {{v[0-9]+}} offset:8
+; GCN: s_endpgm
 define amdgpu_kernel void @atomic_umax_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1
   %idx.0 = add nsw i32 %tid.x, 2
@@ -280,5 +280,143 @@ define amdgpu_kernel void @atomic_umax_shl_base_lds_0(i32 addrspace(1)* %out, i3
   ret void
 }
 
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_lds:
+; GCN: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 3, v0
+; GCN: ds_write_b32 [[SCALE0]], v{{[0-9]+}} offset:32
+
+; GCN: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 4, v0
+; GCN: ds_write_b32 [[SCALE1]], v{{[0-9]+}} offset:64
+define void @shl_add_ptr_combine_2use_lds(i32 %idx) #0 {
+  %idx.add = add nuw i32 %idx, 4
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to i32 addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to i32 addrspace(3)*
+  store volatile i32 9, i32 addrspace(3)* %ptr0
+  store volatile i32 10, i32 addrspace(3)* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_max_lds_offset:
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 3, v0
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 4, v0
+; GCN-DAG: ds_write_b32 [[SCALE0]], v{{[0-9]+}} offset:65528
+; GCN-DAG: v_add_{{[iu]}}32_e32 [[ADD1:v[0-9]+]], vcc, 0x1fff0, [[SCALE1]]
+; GCN: ds_write_b32 [[ADD1]], v{{[0-9]+$}}
+define void @shl_add_ptr_combine_2use_max_lds_offset(i32 %idx) #0 {
+  %idx.add = add nuw i32 %idx, 8191
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to i32 addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to i32 addrspace(3)*
+  store volatile i32 9, i32 addrspace(3)* %ptr0
+  store volatile i32 10, i32 addrspace(3)* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_both_max_lds_offset:
+; GCN: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 0x1000, v0
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 4, [[ADD]]
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 5, [[ADD]]
+; GCN-DAG: ds_write_b32 [[SCALE0]], v{{[0-9]+$}}
+; GCN: ds_write_b32 [[SCALE1]], v{{[0-9]+$}}
+define void @shl_add_ptr_combine_2use_both_max_lds_offset(i32 %idx) #0 {
+  %idx.add = add nuw i32 %idx, 4096
+  %shl0 = shl i32 %idx.add, 4
+  %shl1 = shl i32 %idx.add, 5
+  %ptr0 = inttoptr i32 %shl0 to i32 addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to i32 addrspace(3)*
+  store volatile i32 9, i32 addrspace(3)* %ptr0
+  store volatile i32 10, i32 addrspace(3)* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_private:
+; GCN: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 2, v0
+; GCN: buffer_store_dword v{{[0-9]+}}, [[SCALE0]], s[0:3], s4 offen offset:16
+
+; GCN: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 3, v0
+; GCN: buffer_store_dword v{{[0-9]+}}, [[SCALE1]], s[0:3], s4 offen offset:32
+define void @shl_add_ptr_combine_2use_private(i16 zeroext %idx.arg) #0 {
+  %idx = zext i16 %idx.arg to i32
+  %idx.add = add nuw i32 %idx, 4
+  %shl0 = shl i32 %idx.add, 2
+  %shl1 = shl i32 %idx.add, 3
+  %ptr0 = inttoptr i32 %shl0 to i32*
+  %ptr1 = inttoptr i32 %shl1 to i32*
+  store volatile i32 9, i32* %ptr0
+  store volatile i32 10, i32* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_max_private_offset:
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 3, v0
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 4, v0
+; GCN-DAG: buffer_store_dword v{{[0-9]+}}, [[SCALE0]], s[0:3], s4 offen offset:4088
+; GCN-DAG: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 0x1ff0, [[SCALE1]]
+; GCN: buffer_store_dword v{{[0-9]+}}, [[ADD]], s[0:3], s4 offen{{$}}
+define void @shl_add_ptr_combine_2use_max_private_offset(i16 zeroext %idx.arg) #0 {
+  %idx = zext i16 %idx.arg to i32
+  %idx.add = add nuw i32 %idx, 511
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to i32*
+  %ptr1 = inttoptr i32 %shl1 to i32*
+  store volatile i32 9, i32* %ptr0
+  store volatile i32 10, i32* %ptr1
+  ret void
+}
+; GCN-LABEL: {{^}}shl_add_ptr_combine_2use_both_max_private_offset:
+; GCN: v_add_{{[iu]}}32_e32 [[ADD:v[0-9]+]], vcc, 0x100, v0
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 4, [[ADD]]
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 5, [[ADD]]
+; GCN-DAG: buffer_store_dword v{{[0-9]+}}, [[SCALE0]], s[0:3], s4 offen{{$}}
+; GCN: buffer_store_dword v{{[0-9]+}}, [[SCALE1]], s[0:3], s4 offen{{$}}
+define void @shl_add_ptr_combine_2use_both_max_private_offset(i16 zeroext %idx.arg) #0 {
+  %idx = zext i16 %idx.arg to i32
+  %idx.add = add nuw i32 %idx, 256
+  %shl0 = shl i32 %idx.add, 4
+  %shl1 = shl i32 %idx.add, 5
+  %ptr0 = inttoptr i32 %shl0 to i32*
+  %ptr1 = inttoptr i32 %shl1 to i32*
+  store volatile i32 9, i32* %ptr0
+  store volatile i32 10, i32* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_or_ptr_combine_2use_lds:
+; GCN: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 3, v0
+; GCN: ds_write_b32 [[SCALE0]], v{{[0-9]+}} offset:32
+
+; GCN: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 4, v0
+; GCN: ds_write_b32 [[SCALE1]], v{{[0-9]+}} offset:64
+define void @shl_or_ptr_combine_2use_lds(i32 %idx) #0 {
+  %idx.add = or i32 %idx, 4
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to i32 addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to i32 addrspace(3)*
+  store volatile i32 9, i32 addrspace(3)* %ptr0
+  store volatile i32 10, i32 addrspace(3)* %ptr1
+  ret void
+}
+
+; GCN-LABEL: {{^}}shl_or_ptr_combine_2use_max_lds_offset:
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE0:v[0-9]+]], 3, v0
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALE1:v[0-9]+]], 4, v0
+; GCN-DAG: ds_write_b32 [[SCALE0]], v{{[0-9]+}} offset:65528
+; GCN-DAG: v_or_b32_e32 [[ADD1:v[0-9]+]], 0x1fff0, [[SCALE1]]
+; GCN: ds_write_b32 [[ADD1]], v{{[0-9]+$}}
+define void @shl_or_ptr_combine_2use_max_lds_offset(i32 %idx) #0 {
+  %idx.add = or i32 %idx, 8191
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to i32 addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to i32 addrspace(3)*
+  store volatile i32 9, i32 addrspace(3)* %ptr0
+  store volatile i32 10, i32 addrspace(3)* %ptr1
+  ret void
+}
+
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 14ca635c6dadd..a185bc35ee5d7 100644
--- a/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -6,7 +6,7 @@
 
 ; GCN-LABEL: {{^}}v_test_i32_x_sub_64:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_subrev_i32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
+; GCN: v_subrev_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
 define amdgpu_kernel void @v_test_i32_x_sub_64(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -21,8 +21,8 @@ define amdgpu_kernel void @v_test_i32_x_sub_64(i32 addrspace(1)* %out, i32 addrs
 ; GCN-LABEL: {{^}}v_test_i32_x_sub_64_multi_use:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
 ; GCN: {{buffer|flat}}_load_dword [[Y:v[0-9]+]]
-; GCN-DAG: v_subrev_i32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
-; GCN-DAG: v_subrev_i32_e32 v{{[0-9]+}}, vcc, 64, [[Y]]
+; GCN-DAG: v_subrev_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
+; GCN-DAG: v_subrev_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 64, [[Y]]
 define amdgpu_kernel void @v_test_i32_x_sub_64_multi_use(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -39,7 +39,7 @@ define amdgpu_kernel void @v_test_i32_x_sub_64_multi_use(i32 addrspace(1)* %out,
 
 ; GCN-LABEL: {{^}}v_test_i32_64_sub_x:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
+; GCN: v_sub_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 64, [[X]]
 define amdgpu_kernel void @v_test_i32_64_sub_x(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -53,7 +53,7 @@ define amdgpu_kernel void @v_test_i32_64_sub_x(i32 addrspace(1)* %out, i32 addrs
 
 ; GCN-LABEL: {{^}}v_test_i32_x_sub_65:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, 0xffffffbf, [[X]]
+; GCN: v_add_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 0xffffffbf, [[X]]
 define amdgpu_kernel void @v_test_i32_x_sub_65(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -67,7 +67,7 @@ define amdgpu_kernel void @v_test_i32_x_sub_65(i32 addrspace(1)* %out, i32 addrs
 
 ; GCN-LABEL: {{^}}v_test_i32_65_sub_x:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, 0x41, [[X]]
+; GCN: v_sub_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 0x41, [[X]]
 define amdgpu_kernel void @v_test_i32_65_sub_x(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -81,7 +81,7 @@ define amdgpu_kernel void @v_test_i32_65_sub_x(i32 addrspace(1)* %out, i32 addrs
 
 ; GCN-LABEL: {{^}}v_test_i32_x_sub_neg16:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, 16, [[X]]
+; GCN: v_add_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 16, [[X]]
 define amdgpu_kernel void @v_test_i32_x_sub_neg16(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -95,7 +95,7 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg16(i32 addrspace(1)* %out, i32 ad
 
 ; GCN-LABEL: {{^}}v_test_i32_neg16_sub_x:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, -16, [[X]]
+; GCN: v_sub_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, -16, [[X]]
 define amdgpu_kernel void @v_test_i32_neg16_sub_x(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -109,7 +109,7 @@ define amdgpu_kernel void @v_test_i32_neg16_sub_x(i32 addrspace(1)* %out, i32 ad
 
 ; GCN-LABEL: {{^}}v_test_i32_x_sub_neg17:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, 17, [[X]]
+; GCN: v_add_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 17, [[X]]
 define amdgpu_kernel void @v_test_i32_x_sub_neg17(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
@@ -123,7 +123,7 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg17(i32 addrspace(1)* %out, i32 ad
 
 ; GCN-LABEL: {{^}}v_test_i32_neg17_sub_x:
 ; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]]
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, 0xffffffef, [[X]]
+; GCN: v_sub_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, 0xffffffef, [[X]]
 define amdgpu_kernel void @v_test_i32_neg17_sub_x(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/shrink-carry.mir b/test/CodeGen/AMDGPU/shrink-carry.mir
index ce0cec75403cf..d499b2192e97f 100644
--- a/test/CodeGen/AMDGPU/shrink-carry.mir
+++ b/test/CodeGen/AMDGPU/shrink-carry.mir
@@ -1,7 +1,7 @@
 # RUN: llc -march=amdgcn -verify-machineinstrs -start-before si-shrink-instructions -stop-before si-insert-skips -o - %s | FileCheck -check-prefix=GCN %s
 
 # GCN-LABEL: name: subbrev{{$}}
-# GCN:       V_SUBBREV_U32_e64 0, undef %vgpr0, killed %vcc, implicit %exec
+# GCN:       V_SUBBREV_U32_e64 0, undef %vgpr0, killed renamable %vcc, implicit %exec
 
 ---
 name:            subbrev
@@ -10,9 +10,9 @@ registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
 body:             |
   bb.0:
 
@@ -21,12 +21,11 @@ body:             |
     %2 = IMPLICIT_DEF
     %3 = V_CMP_GT_U32_e64 %0, %1, implicit %exec
     %4, %5 = V_SUBBREV_U32_e64 0, %0, %3, implicit %exec
-    S_ENDPGM
 
 ...
 
 # GCN-LABEL: name: subb{{$}}
-# GCN:       V_SUBB_U32_e64 undef %vgpr0, 0, killed %vcc, implicit %exec
+# GCN:       V_SUBB_U32_e64 undef %vgpr0, 0, killed renamable %vcc, implicit %exec
 
 ---
 name:            subb
@@ -35,9 +34,9 @@ registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
 body:             |
   bb.0:
 
@@ -46,12 +45,11 @@ body:             |
     %2 = IMPLICIT_DEF
     %3 = V_CMP_GT_U32_e64 %0, %1, implicit %exec
     %4, %5 = V_SUBB_U32_e64 %0, 0, %3, implicit %exec
-    S_ENDPGM
 
 ...
 
 # GCN-LABEL: name: addc{{$}}
-# GCN:       V_ADDC_U32_e32 0, undef %vgpr0, implicit-def %vcc, implicit killed %vcc, implicit %exec
+# GCN:       V_ADDC_U32_e32 0, undef renamable %vgpr0, implicit-def %vcc, implicit killed %vcc, implicit %exec
 
 ---
 name:            addc
@@ -60,9 +58,9 @@ registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
 body:             |
   bb.0:
 
@@ -71,12 +69,11 @@ body:             |
     %2 = IMPLICIT_DEF
     %3 = V_CMP_GT_U32_e64 %0, %1, implicit %exec
     %4, %5 = V_ADDC_U32_e64 0, %0, %3, implicit %exec
-    S_ENDPGM
 
 ...
 
 # GCN-LABEL: name: addc2{{$}}
-# GCN:       V_ADDC_U32_e32 0, undef %vgpr0, implicit-def %vcc, implicit killed %vcc, implicit %exec
+# GCN:       V_ADDC_U32_e32 0, undef renamable %vgpr0, implicit-def %vcc, implicit killed %vcc, implicit %exec
 
 ---
 name:            addc2
@@ -85,9 +82,9 @@ registers:
   - { id: 0, class: vgpr_32 }
   - { id: 1, class: vgpr_32 }
   - { id: 2, class: vgpr_32 }
-  - { id: 3, class: sreg_64 }
+  - { id: 3, class: sreg_64_xexec }
   - { id: 4, class: vgpr_32 }
-  - { id: 5, class: sreg_64 }
+  - { id: 5, class: sreg_64_xexec }
 body:             |
   bb.0:
 
@@ -96,6 +93,5 @@ body:             |
     %2 = IMPLICIT_DEF
     %3 = V_CMP_GT_U32_e64 %0, %1, implicit %exec
     %4, %5 = V_ADDC_U32_e64 %0, 0, %3, implicit %exec
-    S_ENDPGM
 
 ...
diff --git a/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir b/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
index 767118eb8d118..0ffee0c4fcf4c 100644
--- a/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
+++ b/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
@@ -8,8 +8,8 @@
 
 ...
 # GCN-LABEL: name: shrink_add_vop3{{$}}
-# GCN: %29, %9 = V_ADD_I32_e64 %19, %17, implicit %exec
-# GCN: %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
+# GCN: %29:vgpr_32, %9:sreg_64_xexec = V_ADD_I32_e64 %19, %17, implicit %exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
 name:            shrink_add_vop3
 alignment:       0
 exposesReturnsTwice: false
@@ -27,7 +27,7 @@ registers:
   - { id: 6, class: sreg_32 }
   - { id: 7, class: sreg_32 }
   - { id: 8, class: sreg_32_xm0 }
-  - { id: 9, class: sreg_64 }
+  - { id: 9, class: sreg_64_xexec }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
@@ -91,8 +91,8 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: shrink_sub_vop3{{$}}
-# GCN: %29, %9 = V_SUB_I32_e64 %19, %17, implicit %exec
-# GCN: %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
+# GCN: %29:vgpr_32, %9:sreg_64_xexec = V_SUB_I32_e64 %19, %17, implicit %exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
 
 name:            shrink_sub_vop3
 alignment:       0
@@ -111,7 +111,7 @@ registers:
   - { id: 6, class: sreg_32 }
   - { id: 7, class: sreg_32 }
   - { id: 8, class: sreg_32_xm0 }
-  - { id: 9, class: sreg_64 }
+  - { id: 9, class: sreg_64_xexec }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
@@ -175,8 +175,8 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: shrink_subrev_vop3{{$}}
-# GCN: %29, %9 = V_SUBREV_I32_e64 %19, %17, implicit %exec
-# GCN: %24 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
+# GCN: %29:vgpr_32, %9:sreg_64_xexec = V_SUBREV_I32_e64 %19, %17, implicit %exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %9, implicit %exec
 
 name:            shrink_subrev_vop3
 alignment:       0
@@ -195,7 +195,7 @@ registers:
   - { id: 6, class: sreg_32 }
   - { id: 7, class: sreg_32 }
   - { id: 8, class: sreg_32_xm0 }
-  - { id: 9, class: sreg_64 }
+  - { id: 9, class: sreg_64_xexec }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
@@ -259,8 +259,8 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: check_addc_src2_vop3{{$}}
-# GCN: %29, %vcc = V_ADDC_U32_e64 %19, %17, %9, implicit %exec
-# GCN: %24 = V_CNDMASK_B32_e64 0, 1, killed %vcc, implicit %exec
+# GCN: %29:vgpr_32, %vcc = V_ADDC_U32_e64 %19, %17, %9, implicit %exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %vcc, implicit %exec
 name: check_addc_src2_vop3
 alignment:       0
 exposesReturnsTwice: false
@@ -278,7 +278,7 @@ registers:
   - { id: 6, class: sreg_32 }
   - { id: 7, class: sreg_32 }
   - { id: 8, class: sreg_32_xm0 }
-  - { id: 9, class: sreg_64 }
+  - { id: 9, class: sreg_64_xexec }
   - { id: 10, class: sreg_32_xm0 }
   - { id: 11, class: sreg_32_xm0 }
   - { id: 12, class: sgpr_64 }
@@ -343,7 +343,7 @@ body:             |
 ...
 ---
 # GCN-LABEL: name: shrink_addc_vop3{{$}}
-# GCN: %29 = V_ADDC_U32_e32 %19, %17, implicit-def %vcc, implicit %vcc, implicit %exec
+# GCN: %29:vgpr_32 = V_ADDC_U32_e32 %19, %17, implicit-def %vcc, implicit %vcc, implicit %exec
 # GCN %24 = V_CNDMASK_B32_e64 0, 1, killed %vcc, implicit %exec
 
 name:            shrink_addc_vop3
@@ -429,8 +429,8 @@ body:             |
 
 ---
 # GCN-LABEL: name: shrink_addc_undef_vcc{{$}}
-# GCN: %29 = V_ADDC_U32_e32 %19, %17, implicit-def %vcc, implicit undef %vcc, implicit %exec
-# GCN: %24 = V_CNDMASK_B32_e64 0, 1, killed %vcc, implicit %exec
+# GCN: %29:vgpr_32 = V_ADDC_U32_e32 %19, %17, implicit-def %vcc, implicit undef %vcc, implicit %exec
+# GCN: %24:vgpr_32 = V_CNDMASK_B32_e64 0, 1, killed %vcc, implicit %exec
 name:            shrink_addc_undef_vcc
 alignment:       0
 exposesReturnsTwice: false
diff --git a/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll b/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll
index 5c6663dbbdab9..a99d18147ccf1 100644
--- a/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll
+++ b/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll
@@ -42,7 +42,6 @@ bb5:                                              ; preds = %bb3, %bb1
 ; GCN: s_and_saveexec_b64
 ; GCN: ; mask branch [[UNIFIED_RET:BB[0-9]+_[0-9]+]]
 ; GCN-NEXT: [[UNIFIED_RET]]:
-; GCN-NEXT: s_or_b64 exec, exec
 ; GCN-NEXT: s_endpgm
 ; GCN: .Lfunc_end
 define amdgpu_kernel void @annotate_ret_noloop(<4 x float> addrspace(1)* noalias nocapture readonly %arg) #0 {
diff --git a/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll b/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll
index 580268deb85d1..7ae4636a0b5df 100644
--- a/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll
+++ b/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll
@@ -3,7 +3,7 @@
 ; register operands in the correct order when modifying the opcode of an
 ; instruction to V_ADD_I32_e32.
 
-; CHECK: %{{[0-9]+}} = V_ADD_I32_e32 %{{[0-9]+}}, %{{[0-9]+}}, implicit-def %vcc, implicit %exec
+; CHECK: %{{[0-9]+}}:vgpr_32 = V_ADD_I32_e32 %{{[0-9]+}}, %{{[0-9]+}}, implicit-def %vcc, implicit %exec
 
 define amdgpu_kernel void @test(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
 entry:
diff --git a/test/CodeGen/AMDGPU/si-lower-control-flow-kill.ll b/test/CodeGen/AMDGPU/si-lower-control-flow-kill.ll
new file mode 100644
index 0000000000000..d422510d1cafe
--- /dev/null
+++ b/test/CodeGen/AMDGPU/si-lower-control-flow-kill.ll
@@ -0,0 +1,71 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}if_with_kill:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]],
+; GCN-NEXT: s_xor_b64 s[{{[0-9:]+}}], exec, [[SAVEEXEC]]
+define amdgpu_ps void @if_with_kill(i32 %arg) {
+.entry:
+  %cmp = icmp eq i32 %arg, 32
+  br i1 %cmp, label %then, label %endif
+
+then:
+  tail call void @llvm.AMDGPU.kilp()
+  br label %endif
+
+endif:
+  ret void
+}
+
+; GCN-LABEL: {{^}}if_with_loop_kill_after:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]],
+; GCN-NEXT: s_xor_b64 s[{{[0-9:]+}}], exec, [[SAVEEXEC]]
+define amdgpu_ps void @if_with_loop_kill_after(i32 %arg) {
+.entry:
+  %cmp = icmp eq i32 %arg, 32
+  br i1 %cmp, label %then, label %endif
+
+then:
+  %sub = sub i32 %arg, 1
+  br label %loop
+
+loop:
+  %ind = phi i32 [%sub, %then], [%dec, %loop]
+  %dec = sub i32 %ind, 1
+  %cc = icmp ne i32 %ind, 0
+  br i1 %cc, label %loop, label %break
+
+break:
+  tail call void @llvm.AMDGPU.kilp()
+  br label %endif
+
+endif:
+  ret void
+}
+
+; GCN-LABEL: {{^}}if_with_kill_inside_loop:
+; GCN:      s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9:]+\]]],
+; GCN-NEXT: s_xor_b64 s[{{[0-9:]+}}], exec, [[SAVEEXEC]]
+define amdgpu_ps void @if_with_kill_inside_loop(i32 %arg) {
+.entry:
+  %cmp = icmp eq i32 %arg, 32
+  br i1 %cmp, label %then, label %endif
+
+then:
+  %sub = sub i32 %arg, 1
+  br label %loop
+
+loop:
+  %ind = phi i32 [%sub, %then], [%dec, %loop]
+  %dec = sub i32 %ind, 1
+  %cc = icmp ne i32 %ind, 0
+  tail call void @llvm.AMDGPU.kilp()
+  br i1 %cc, label %loop, label %break
+
+break:
+  br label %endif
+
+endif:
+  ret void
+}
+
+declare void @llvm.AMDGPU.kilp()
diff --git a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll
index 5b0d5274d5bc1..ce85a66634044 100644
--- a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll
+++ b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll
@@ -3,7 +3,6 @@
 ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator:
 ; GCN: v_cmp_eq_u32
 ; GCN: s_and_saveexec_b64
-; GCN: s_xor_b64
 ; GCN: ; mask branch [[RET:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT: BB{{[0-9]+_[0-9]+}}: ; %unreachable
@@ -11,7 +10,6 @@
 ; GCN: ; divergent unreachable
 
 ; GCN-NEXT: [[RET]]: ; %UnifiedReturnBlock
-; GCN-NEXT: s_or_b64 exec, exec
 ; GCN: s_endpgm
 
 define amdgpu_kernel void @lower_control_flow_unreachable_terminator() #0 {
@@ -31,7 +29,6 @@ ret:
 ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator_swap_block_order:
 ; GCN: v_cmp_ne_u32
 ; GCN: s_and_saveexec_b64
-; GCN: s_xor_b64
 ; GCN: ; mask branch [[RETURN:BB[0-9]+_[0-9]+]]
 
 ; GCN-NEXT: {{^BB[0-9]+_[0-9]+}}: ; %unreachable
@@ -39,7 +36,6 @@ ret:
 ; GCN: ; divergent unreachable
 
 ; GCN: [[RETURN]]:
-; GCN-NEXT: s_or_b64 exec, exec
 ; GCN-NEXT: s_endpgm
 define amdgpu_kernel void @lower_control_flow_unreachable_terminator_swap_block_order() #0 {
 bb:
@@ -59,7 +55,7 @@ unreachable:
 ; GCN: s_cmp_lg_u32
 ; GCN: s_cbranch_scc0 [[UNREACHABLE:BB[0-9]+_[0-9]+]]
 
-; GCN-NEXT: BB#{{[0-9]+}}: ; %ret
+; GCN-NEXT: %bb.{{[0-9]+}}: ; %ret
 ; GCN-NEXT: s_endpgm
 
 ; GCN: [[UNREACHABLE]]:
diff --git a/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index 17109187d5387..6d2f89ab96da5 100644
--- a/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -1,4 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -check-prefix=FUNC -check-prefix=CI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -enable-amdgpu-aa=0 -verify-machineinstrs -enable-misched -enable-aa-sched-mi < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
 declare void @llvm.amdgcn.tbuffer.store.v4i32(<4 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1)
@@ -10,9 +11,13 @@ declare i32 @llvm.amdgcn.workitem.id.x() #2
 @stored_constant_ptr = addrspace(3) global i32 addrspace(2)* undef, align 8
 @stored_global_ptr = addrspace(3) global i32 addrspace(1)* undef, align 8
 
-; FUNC-LABEL: @reorder_local_load_global_store_local_load
+; GCN-LABEL: {{^}}reorder_local_load_global_store_local_load:
 ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3
 ; CI: buffer_store_dword
+
+; GFX9: global_store_dword
+; GFX9: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:1 offset1:3
+; GFX9: global_store_dword
 define amdgpu_kernel void @reorder_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
   %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
 
@@ -29,10 +34,14 @@ define amdgpu_kernel void @reorder_local_load_global_store_local_load(i32 addrsp
   ret void
 }
 
-; FUNC-LABEL: @no_reorder_local_load_volatile_global_store_local_load
+; GCN-LABEL: {{^}}no_reorder_local_load_volatile_global_store_local_load:
 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
 ; CI: buffer_store_dword
 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
+
+; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
+; GFX9: global_store_dword
+; GFX9: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
 define amdgpu_kernel void @no_reorder_local_load_volatile_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
   %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
 
@@ -49,10 +58,16 @@ define amdgpu_kernel void @no_reorder_local_load_volatile_global_store_local_loa
   ret void
 }
 
-; FUNC-LABEL: @no_reorder_barrier_local_load_global_store_local_load
+; GCN-LABEL: {{^}}no_reorder_barrier_local_load_global_store_local_load:
 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
 ; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
 ; CI: buffer_store_dword
+
+; GFX9-DAG: global_store_dword
+; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:4
+; GFX9: s_barrier
+; GFX9-DAG: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12
+; GFX9-DAG: global_store_dword
 define amdgpu_kernel void @no_reorder_barrier_local_load_global_store_local_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
   %ptr0 = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(3)* @stored_lds_ptr, align 4
 
@@ -70,13 +85,20 @@ define amdgpu_kernel void @no_reorder_barrier_local_load_global_store_local_load
   ret void
 }
 
-; FUNC-LABEL: @reorder_constant_load_global_store_constant_load
-; CI-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
-; CI: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
+; GCN-LABEL: {{^}}reorder_constant_load_global_store_constant_load:
+; GCN-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
+; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
 ; CI: buffer_store_dword
+
 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1
 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3
+
+; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4
+; GFX9: global_store_dword
+; GFX9: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc
+
 ; CI: buffer_store_dword
+; GFX9: global_store_dword
 define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(1)* %gptr) #0 {
   %ptr0 = load i32 addrspace(2)*, i32 addrspace(2)* addrspace(3)* @stored_constant_ptr, align 8
 
@@ -93,13 +115,19 @@ define amdgpu_kernel void @reorder_constant_load_global_store_constant_load(i32
   ret void
 }
 
-; FUNC-LABEL: @reorder_constant_load_local_store_constant_load
-; CI: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
-; CI: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
+; GCN-LABEL: {{^}}reorder_constant_load_local_store_constant_load:
+; GCN: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}}
+; GCN: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}}
+
 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1
 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3
-; CI: ds_write_b32
+
+; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x4
+; GFX9-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0xc
+
+; GCN: ds_write_b32
 ; CI: buffer_store_dword
+; GFX9: global_store_dword
 define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr) #0 {
   %ptr0 = load i32 addrspace(2)*, i32 addrspace(2)* addrspace(3)* @stored_constant_ptr, align 8
 
@@ -116,12 +144,13 @@ define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(i32 a
   ret void
 }
 
-; FUNC-LABEL: @reorder_smrd_load_local_store_smrd_load
-; CI: s_load_dword
-; CI: s_load_dword
-; CI: s_load_dword
-; CI: ds_write_b32
+; GCN-LABEL: {{^}}reorder_smrd_load_local_store_smrd_load:
+; GCN: s_load_dword
+; GCN: s_load_dword
+; GCN: s_load_dword
+; GCN: ds_write_b32
 ; CI: buffer_store_dword
+; GFX9: global_store_dword
 define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(i32 addrspace(1)* %out, i32 addrspace(3)* noalias %lptr, i32 addrspace(2)* %ptr0) #0 {
   %ptr1 = getelementptr inbounds i32, i32 addrspace(2)* %ptr0, i64 1
   %ptr2 = getelementptr inbounds i32, i32 addrspace(2)* %ptr0, i64 2
@@ -136,11 +165,15 @@ define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(i32 addrspace
   ret void
 }
 
-; FUNC-LABEL: @reorder_global_load_local_store_global_load
+; GCN-LABEL: {{^}}reorder_global_load_local_store_global_load:
 ; CI: ds_write_b32
 ; CI: buffer_load_dword
 ; CI: buffer_load_dword
 ; CI: buffer_store_dword
+
+; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:4
+; GFX9: global_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:12
+; GFX9: ds_write_b32
 define amdgpu_kernel void @reorder_global_load_local_store_global_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr, i32 addrspace(1)* %ptr0) #0 {
   %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 1
   %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 3
@@ -155,12 +188,13 @@ define amdgpu_kernel void @reorder_global_load_local_store_global_load(i32 addrs
   ret void
 }
 
-; FUNC-LABEL: @reorder_local_offsets
-; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:100 offset1:102
-; CI-DAG: ds_write2_b32 {{v[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:3 offset1:100
-; CI-DAG: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:408
+; GCN-LABEL: {{^}}reorder_local_offsets:
+; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:100 offset1:102
+; GCN-DAG: ds_write2_b32 {{v[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:3 offset1:100
+; GCN-DAG: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:408
 ; CI: buffer_store_dword
-; CI: s_endpgm
+; GFX9: global_store_dword
+; GCN: s_endpgm
 define amdgpu_kernel void @reorder_local_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(3)* noalias nocapture %ptr0) #0 {
   %ptr1 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 3
   %ptr2 = getelementptr inbounds i32, i32 addrspace(3)* %ptr0, i32 100
@@ -179,7 +213,7 @@ define amdgpu_kernel void @reorder_local_offsets(i32 addrspace(1)* nocapture %ou
   ret void
 }
 
-; FUNC-LABEL: @reorder_global_offsets
+; GCN-LABEL: {{^}}reorder_global_offsets:
 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400
 ; CI-DAG: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408
 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12
@@ -187,6 +221,14 @@ define amdgpu_kernel void @reorder_local_offsets(i32 addrspace(1)* nocapture %ou
 ; CI-DAG: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408
 ; CI: buffer_store_dword
 ; CI: s_endpgm
+
+; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:400
+; GFX9-DAG: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:408
+; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:12
+; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:400
+; GFX9-DAG: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, off offset:408
+; GFX9: global_store_dword
+; GFX9: s_endpgm
 define amdgpu_kernel void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(1)* noalias nocapture %ptr0) #0 {
   %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3
   %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 100
@@ -205,22 +247,34 @@ define amdgpu_kernel void @reorder_global_offsets(i32 addrspace(1)* nocapture %o
   ret void
 }
 
-; FUNC-LABEL: {{^}}reorder_global_offsets_addr64_soffset0:
-; GCN: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:12{{$}}
-; GCN-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:28{{$}}
-; GCN-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:44{{$}}
+; GCN-LABEL: {{^}}reorder_global_offsets_addr64_soffset0:
+; CI:      buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:12{{$}}
+; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:28{{$}}
+; CI-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:44{{$}}
+
+; CI: v_mov_b32
+; CI: v_mov_b32
+
+; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
+
+; CI: v_add_i32
+; CI: v_add_i32
+
+; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:20{{$}}
+
+; CI: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:36{{$}}
+; CI-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:52{{$}}
 
-; GCN: v_mov_b32
-; GCN: v_mov_b32
 
-; GCN: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64{{$}}
-; GCN-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:20{{$}}
+; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:12
+; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:28
+; GFX9: global_load_dword {{v[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, off offset:44
 
-; GCN: v_add_i32
-; GCN: v_add_i32
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off{{$}}
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:20
 
-; GCN: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:36{{$}}
-; GCN-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:52{{$}}
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:36
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, off offset:52
 define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(i32 addrspace(1)* noalias nocapture %ptr.base) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %id.ext = sext i32 %id to i64
@@ -245,7 +299,7 @@ define amdgpu_kernel void @reorder_global_offsets_addr64_soffset0(i32 addrspace(
   ret void
 }
 
-; XFUNC-LABEL: @reorder_local_load_tbuffer_store_local_load
+; XGCN-LABEL: {{^}}reorder_local_load_tbuffer_store_local_load:
 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x4
 ; XCI: TBUFFER_STORE_FORMAT
 ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x8
diff --git a/test/CodeGen/AMDGPU/sibling-call.ll b/test/CodeGen/AMDGPU/sibling-call.ll
new file mode 100644
index 0000000000000..f7e8a1d80e9ba
--- /dev/null
+++ b/test/CodeGen/AMDGPU/sibling-call.ll
@@ -0,0 +1,272 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=fiji -mattr=-flat-for-global -enable-ipra=0 -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,CIVI,MESA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=hawaii -enable-ipra=0 -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI,CIVI,MESA %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa-amdgiz -mcpu=gfx900 -mattr=-flat-for-global -enable-ipra=0 -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,MESA %s
+target datalayout = "A5"
+
+; FIXME: Why is this commuted only sometimes?
+; GCN-LABEL: {{^}}i32_fastcc_i32_i32:
+; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GCN-NEXT: s_setpc_b64
+define fastcc i32 @i32_fastcc_i32_i32(i32 %arg0, i32 %arg1) #1 {
+  %add0 = add i32 %arg0, %arg1
+  ret i32 %add0
+}
+
+; GCN-LABEL: {{^}}i32_fastcc_i32_i32_stack_object:
+; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s5 offset:24
+; GCN: s_waitcnt vmcnt(0)
+; GCN: s_setpc_b64
+; GCN: ; ScratchSize: 68
+define fastcc i32 @i32_fastcc_i32_i32_stack_object(i32 %arg0, i32 %arg1) #1 {
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %add0 = add i32 %arg0, %arg1
+  ret i32 %add0
+}
+
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32:
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_stack_object:
+; GCN: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
+; GCN: buffer_store_dword [[NINE]], off, s[0:3], s5 offset:24
+; GCN: s_setpc_b64
+; GCN: ; ScratchSize: 68
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32_stack_object(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_callee_stack_object:
+; GCN: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
+; GCN: buffer_store_dword [[NINE]], off, s[0:3], s5 offset:24
+; GCN: s_setpc_b64
+; GCN: ; ScratchSize: 136
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32_callee_stack_object(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_stack_object(i32 %a, i32 %b)
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_unused_result:
+define fastcc void @sibling_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
+  ret void
+}
+
+; It doesn't make sense to do a tail from a kernel
+; GCN-LABEL: {{^}}kernel_call_i32_fastcc_i32_i32_unused_result:
+;define amdgpu_kernel void @kernel_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
+define amdgpu_kernel void @kernel_call_i32_fastcc_i32_i32_unused_result(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
+  ret void
+}
+
+; GCN-LABEL: {{^}}i32_fastcc_i32_byval_i32:
+; GCN: s_waitcnt
+; GCN-NEXT: s_mov_b32 s5, s32
+; GCN-NEXT: buffer_load_dword v1, off, s[0:3], s5 offset:4
+; GCN-NEXT: s_waitcnt vmcnt(0)
+
+; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v1, v0
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+
+; GCN-NEXT: s_setpc_b64 s[30:31]
+define fastcc i32 @i32_fastcc_i32_byval_i32(i32 %arg0, i32 addrspace(5)* byval align 4 %arg1) #1 {
+  %arg1.load = load i32, i32 addrspace(5)* %arg1, align 4
+  %add0 = add i32 %arg0, %arg1.load
+  ret i32 %add0
+}
+
+; Tail call disallowed with byval in parent.
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_byval_i32_byval_parent:
+; GCN-NOT: v_writelane_b32 v{{[0-9]+}}, s32
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[0:3], s32 offset:4
+; GCN: s_swappc_b64
+; GCN-NOT: v_readlane_b32 s32
+; GCN: s_setpc_b64
+define fastcc i32 @sibling_call_i32_fastcc_i32_byval_i32_byval_parent(i32 %a, i32 addrspace(5)* byval %b.byval, i32 %c) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_byval_i32(i32 %a, i32 addrspace(5)* %b.byval)
+  ret i32 %ret
+}
+
+; Tail call disallowed with byval in parent, not callee.
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_byval_i32:
+; GCN-NOT: v0
+; GCN-NOT: s32
+; GCN: buffer_load_dword v1, off, s[0:3], s4 offset:16
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v1, off, s[0:3], s5 offset:4
+; GCN-NEXT: s_setpc_b64
+define fastcc i32 @sibling_call_i32_fastcc_i32_byval_i32(i32 %a, [16 x i32] %large) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_byval_i32(i32 %a, i32 addrspace(5)* inttoptr (i32 16 to i32 addrspace(5)*))
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}i32_fastcc_i32_i32_a32i32:
+; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-DAG: buffer_load_dword [[LOAD_0:v[0-9]+]], off, s[0:3], s5 offset:4
+; GCN-DAG: buffer_load_dword [[LOAD_1:v[0-9]+]], off, s[0:3], s5 offset:8
+
+; CIVI-NEXT: v_add_{{i|u}}32_e32 v0, vcc, v1, v0
+; CIVI: v_add_{{i|u}}32_e32 v0, vcc, [[LOAD_0]], v0
+; CIVI: v_add_{{i|u}}32_e32 v0, vcc, [[LOAD_1]], v0
+
+
+; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
+; GFX9: v_add_u32_e32 v0, v0, [[LOAD_0]]
+; GFX9: v_add_u32_e32 v0, v0, [[LOAD_1]]
+
+; GCN-NEXT: s_setpc_b64
+define fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %arg0, i32 %arg1, [32 x i32] %large) #1 {
+  %val_firststack = extractvalue [32 x i32] %large, 30
+  %val_laststack = extractvalue [32 x i32] %large, 31
+  %add0 = add i32 %arg0, %arg1
+  %add1 = add i32 %add0, %val_firststack
+  %add2 = add i32 %add1, %val_laststack
+  ret i32 %add2
+}
+
+; FIXME: Why load and store same location for stack args?
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_a32i32:
+; GCN: s_mov_b32 s5, s32
+
+; GCN-DAG: buffer_store_dword v32, off, s[0:3], s5 offset:16 ; 4-byte Folded Spill
+; GCN-DAG: buffer_store_dword v33, off, s[0:3], s5 offset:12 ; 4-byte Folded Spill
+
+; GCN-DAG: buffer_load_dword [[LOAD_0:v[0-9]+]], off, s[0:3], s5 offset:4
+; GCN-DAG: buffer_load_dword [[LOAD_1:v[0-9]+]], off, s[0:3], s5 offset:8
+
+; GCN-NOT: s32
+
+; GCN-DAG: buffer_store_dword [[LOAD_0]], off, s[0:3], s5 offset:4
+; GCN-DAG: buffer_store_dword [[LOAD_1]], off, s[0:3], s5 offset:8
+
+; GCN-DAG: buffer_load_dword v32, off, s[0:3], s5 offset:16 ; 4-byte Folded Reload
+; GCN-DAG: buffer_load_dword v33, off, s[0:3], s5 offset:12 ; 4-byte Folded Reload
+
+; GCN-NOT: s32
+; GCN: s_setpc_b64
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c)
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_a32i32_stack_object:
+; GCN-DAG: s_mov_b32 s5, s32
+; GCN-NOT: s32
+; GCN-DAG: v_mov_b32_e32 [[NINE:v[0-9]+]], 9
+; GCN: buffer_store_dword [[NINE]], off, s[0:3], s5 offset:44
+
+; GCN-NOT: s32
+; GCN: s_setpc_b64
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32_a32i32_stack_object(i32 %a, i32 %b, [32 x i32] %c) #1 {
+entry:
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c)
+  ret i32 %ret
+}
+
+; If the callee requires more stack argument space than the caller,
+; don't do a tail call.
+; TODO: Do we really need this restriction?
+
+; GCN-LABEL: {{^}}no_sibling_call_callee_more_stack_space:
+; GCN: s_swappc_b64
+; GCN: s_setpc_b64
+define fastcc i32 @no_sibling_call_callee_more_stack_space(i32 %a, i32 %b) #1 {
+entry:
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] zeroinitializer)
+  ret i32 %ret
+}
+
+; Have another non-tail in the function
+; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32_other_call:
+; GCN: s_mov_b32 s5, s32
+; GCN: buffer_store_dword v34, off, s[0:3], s5 offset:12
+; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:8 ; 4-byte Folded Spill
+; GCN: buffer_store_dword v33, off, s[0:3], s5 offset:4 ; 4-byte Folded Spill
+; GCN-DAG: v_writelane_b32 v34, s33, 0
+; GCN-DAG: v_writelane_b32 v34, s34, 1
+; GCN-DAG: v_writelane_b32 v34, s35, 2
+; GCN-DAG: s_add_u32 s32, s32, 0x400
+
+; GCN: s_getpc_b64
+; GCN: s_swappc_b64
+
+; GCN: s_getpc_b64 s[6:7]
+; GCN: s_add_u32 s6, s6, sibling_call_i32_fastcc_i32_i32@rel32@lo+4
+; GCN: s_addc_u32 s7, s7, sibling_call_i32_fastcc_i32_i32@rel32@hi+4
+
+; GCN-DAG: v_readlane_b32 s33, v34, 0
+; GCN-DAG: v_readlane_b32 s34, v34, 1
+; GCN-DAG: v_readlane_b32 s35, v34, 2
+
+; GCN: buffer_load_dword v33, off, s[0:3], s5 offset:4
+; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:8
+; GCN: buffer_load_dword v34, off, s[0:3], s5 offset:12
+; GCN: s_sub_u32 s32, s32, 0x400
+; GCN: s_setpc_b64 s[6:7]
+define fastcc i32 @sibling_call_i32_fastcc_i32_i32_other_call(i32 %a, i32 %b, i32 %c) #1 {
+entry:
+  %other.call = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
+  %ret = tail call fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %other.call)
+  ret i32 %ret
+}
+
+; Have stack object in caller and stack passed arguments. SP should be
+; in same place at function exit.
+
+; GCN-LABEL: {{^}}sibling_call_stack_objecti32_fastcc_i32_i32_a32i32:
+; GCN: s_mov_b32 s5, s32
+; GCN-NOT: s32
+; GCN: s_setpc_b64 s[6:7]
+define fastcc i32 @sibling_call_stack_objecti32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c) #1 {
+entry:
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] %c)
+  ret i32 %ret
+}
+
+; GCN-LABEL: {{^}}sibling_call_stack_objecti32_fastcc_i32_i32_a32i32_larger_arg_area:
+; GCN: s_mov_b32 s5, s32
+; GCN-NOT: s32
+; GCN: s_setpc_b64 s[6:7]
+define fastcc i32 @sibling_call_stack_objecti32_fastcc_i32_i32_a32i32_larger_arg_area(i32 %a, i32 %b, [36 x i32] %c) #1 {
+entry:
+  %alloca = alloca [16 x i32], align 4, addrspace(5)
+  %gep = getelementptr inbounds [16 x i32], [16 x i32] addrspace(5)* %alloca, i32 0, i32 5
+  store volatile i32 9, i32 addrspace(5)* %gep
+  %ret = tail call fastcc i32 @i32_fastcc_i32_i32_a32i32(i32 %a, i32 %b, [32 x i32] zeroinitializer)
+  ret i32 %ret
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind noinline }
diff --git a/test/CodeGen/AMDGPU/simplify-libcalls.ll b/test/CodeGen/AMDGPU/simplify-libcalls.ll
new file mode 100644
index 0000000000000..47e7d4c9cc22a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/simplify-libcalls.ll
@@ -0,0 +1,787 @@
+; RUN: opt -S -O1 -mtriple=amdgcn---amdgiz -amdgpu-simplify-libcall < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-POSTLINK %s
+; RUN: opt -S -O1 -mtriple=amdgcn---amdgiz -amdgpu-simplify-libcall -amdgpu-prelink  <%s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-PRELINK %s
+; RUN: opt -S -O1 -mtriple=amdgcn---amdgiz -amdgpu-use-native -amdgpu-prelink < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=GCN-NATIVE %s
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos
+; GCN-POSTLINK: tail call fast float @_Z3sinf(
+; GCN-POSTLINK: tail call fast float @_Z3cosf(
+; GCN-PRELINK: call fast float @_Z6sincosfPf(
+; GCN-NATIVE: tail call fast float @_Z10native_sinf(
+; GCN-NATIVE: tail call fast float @_Z10native_cosf(
+define amdgpu_kernel void @test_sincos(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3sinf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  %call2 = tail call fast float @_Z3cosf(float %tmp)
+  %arrayidx3 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  store float %call2, float addrspace(1)* %arrayidx3, align 4
+  ret void
+}
+
+declare float @_Z3sinf(float)
+
+declare float @_Z3cosf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v2
+; GCN-POSTLINK: tail call fast <2 x float> @_Z3sinDv2_f(
+; GCN-POSTLINK: tail call fast <2 x float> @_Z3cosDv2_f(
+; GCN-PRELINK: call fast <2 x float> @_Z6sincosDv2_fPS_(
+; GCN-NATIVE: tail call fast <2 x float> @_Z10native_sinDv2_f(
+; GCN-NATIVE: tail call fast <2 x float> @_Z10native_cosDv2_f(
+define amdgpu_kernel void @test_sincos_v2(<2 x float> addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load <2 x float>, <2 x float> addrspace(1)* %a, align 8
+  %call = tail call fast <2 x float> @_Z3sinDv2_f(<2 x float> %tmp)
+  store <2 x float> %call, <2 x float> addrspace(1)* %a, align 8
+  %call2 = tail call fast <2 x float> @_Z3cosDv2_f(<2 x float> %tmp)
+  %arrayidx3 = getelementptr inbounds <2 x float>, <2 x float> addrspace(1)* %a, i64 1
+  store <2 x float> %call2, <2 x float> addrspace(1)* %arrayidx3, align 8
+  ret void
+}
+
+declare <2 x float> @_Z3sinDv2_f(<2 x float>)
+
+declare <2 x float> @_Z3cosDv2_f(<2 x float>)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v3
+; GCN-POSTLINK: tail call fast <3 x float> @_Z3sinDv3_f(
+; GCN-POSTLINK: tail call fast <3 x float> @_Z3cosDv3_f(
+; GCN-PRELINK: call fast <3 x float> @_Z6sincosDv3_fPS_(
+; GCN-NATIVE: tail call fast <3 x float> @_Z10native_sinDv3_f(
+; GCN-NATIVE: tail call fast <3 x float> @_Z10native_cosDv3_f(
+define amdgpu_kernel void @test_sincos_v3(<3 x float> addrspace(1)* nocapture %a) {
+entry:
+  %castToVec4 = bitcast <3 x float> addrspace(1)* %a to <4 x float> addrspace(1)*
+  %loadVec4 = load <4 x float>, <4 x float> addrspace(1)* %castToVec4, align 16
+  %extractVec4 = shufflevector <4 x float> %loadVec4, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  %call = tail call fast <3 x float> @_Z3sinDv3_f(<3 x float> %extractVec4)
+  %extractVec6 = shufflevector <3 x float> %call, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  store <4 x float> %extractVec6, <4 x float> addrspace(1)* %castToVec4, align 16
+  %call11 = tail call fast <3 x float> @_Z3cosDv3_f(<3 x float> %extractVec4)
+  %arrayidx12 = getelementptr inbounds <3 x float>, <3 x float> addrspace(1)* %a, i64 1
+  %extractVec13 = shufflevector <3 x float> %call11, <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  %storetmp14 = bitcast <3 x float> addrspace(1)* %arrayidx12 to <4 x float> addrspace(1)*
+  store <4 x float> %extractVec13, <4 x float> addrspace(1)* %storetmp14, align 16
+  ret void
+}
+
+declare <3 x float> @_Z3sinDv3_f(<3 x float>)
+
+declare <3 x float> @_Z3cosDv3_f(<3 x float>)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v4
+; GCN-POSTLINK: tail call fast <4 x float> @_Z3sinDv4_f(
+; GCN-POSTLINK: tail call fast <4 x float> @_Z3cosDv4_f(
+; GCN-PRELINK: call fast <4 x float> @_Z6sincosDv4_fPS_(
+; GCN-NATIVE: tail call fast <4 x float> @_Z10native_sinDv4_f(
+; GCN-NATIVE: tail call fast <4 x float> @_Z10native_cosDv4_f(
+define amdgpu_kernel void @test_sincos_v4(<4 x float> addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load <4 x float>, <4 x float> addrspace(1)* %a, align 16
+  %call = tail call fast <4 x float> @_Z3sinDv4_f(<4 x float> %tmp)
+  store <4 x float> %call, <4 x float> addrspace(1)* %a, align 16
+  %call2 = tail call fast <4 x float> @_Z3cosDv4_f(<4 x float> %tmp)
+  %arrayidx3 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %a, i64 1
+  store <4 x float> %call2, <4 x float> addrspace(1)* %arrayidx3, align 16
+  ret void
+}
+
+declare <4 x float> @_Z3sinDv4_f(<4 x float>)
+
+declare <4 x float> @_Z3cosDv4_f(<4 x float>)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v8
+; GCN-POSTLINK: tail call fast <8 x float> @_Z3sinDv8_f(
+; GCN-POSTLINK: tail call fast <8 x float> @_Z3cosDv8_f(
+; GCN-PRELINK: call fast <8 x float> @_Z6sincosDv8_fPS_(
+; GCN-NATIVE: tail call fast <8 x float> @_Z10native_sinDv8_f(
+; GCN-NATIVE: tail call fast <8 x float> @_Z10native_cosDv8_f(
+define amdgpu_kernel void @test_sincos_v8(<8 x float> addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load <8 x float>, <8 x float> addrspace(1)* %a, align 32
+  %call = tail call fast <8 x float> @_Z3sinDv8_f(<8 x float> %tmp)
+  store <8 x float> %call, <8 x float> addrspace(1)* %a, align 32
+  %call2 = tail call fast <8 x float> @_Z3cosDv8_f(<8 x float> %tmp)
+  %arrayidx3 = getelementptr inbounds <8 x float>, <8 x float> addrspace(1)* %a, i64 1
+  store <8 x float> %call2, <8 x float> addrspace(1)* %arrayidx3, align 32
+  ret void
+}
+
+declare <8 x float> @_Z3sinDv8_f(<8 x float>)
+
+declare <8 x float> @_Z3cosDv8_f(<8 x float>)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_sincos_v16
+; GCN-POSTLINK: tail call fast <16 x float> @_Z3sinDv16_f(
+; GCN-POSTLINK: tail call fast <16 x float> @_Z3cosDv16_f(
+; GCN-PRELINK: call fast <16 x float> @_Z6sincosDv16_fPS_(
+; GCN-NATIVE: tail call fast <16 x float> @_Z10native_sinDv16_f(
+; GCN-NATIVE: tail call fast <16 x float> @_Z10native_cosDv16_f(
+define amdgpu_kernel void @test_sincos_v16(<16 x float> addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load <16 x float>, <16 x float> addrspace(1)* %a, align 64
+  %call = tail call fast <16 x float> @_Z3sinDv16_f(<16 x float> %tmp)
+  store <16 x float> %call, <16 x float> addrspace(1)* %a, align 64
+  %call2 = tail call fast <16 x float> @_Z3cosDv16_f(<16 x float> %tmp)
+  %arrayidx3 = getelementptr inbounds <16 x float>, <16 x float> addrspace(1)* %a, i64 1
+  store <16 x float> %call2, <16 x float> addrspace(1)* %arrayidx3, align 64
+  ret void
+}
+
+declare <16 x float> @_Z3sinDv16_f(<16 x float>)
+
+declare <16 x float> @_Z3cosDv16_f(<16 x float>)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_native_recip
+; GCN: store float 0x3FD5555560000000, float addrspace(1)* %a
+define amdgpu_kernel void @test_native_recip(float addrspace(1)* nocapture %a) {
+entry:
+  %call = tail call fast float @_Z12native_recipf(float 3.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z12native_recipf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_half_recip
+; GCN: store float 0x3FD5555560000000, float addrspace(1)* %a
+define amdgpu_kernel void @test_half_recip(float addrspace(1)* nocapture %a) {
+entry:
+  %call = tail call fast float @_Z10half_recipf(float 3.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z10half_recipf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_native_divide
+; GCN: fmul fast float %tmp, 0x3FD5555560000000
+define amdgpu_kernel void @test_native_divide(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z13native_divideff(float %tmp, float 3.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z13native_divideff(float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_half_divide
+; GCN: fmul fast float %tmp, 0x3FD5555560000000
+define amdgpu_kernel void @test_half_divide(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z11half_divideff(float %tmp, float 3.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z11half_divideff(float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_0f
+; GCN: store float 1.000000e+00, float addrspace(1)* %a
+define amdgpu_kernel void @test_pow_0f(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 0.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3powff(float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_0i
+; GCN: store float 1.000000e+00, float addrspace(1)* %a
+define amdgpu_kernel void @test_pow_0i(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 0.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_1f
+; GCN: %tmp = load float, float addrspace(1)* %arrayidx, align 4
+; GCN: store float %tmp, float addrspace(1)* %a, align 4
+define amdgpu_kernel void @test_pow_1f(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 1.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_1i
+; GCN: %tmp = load float, float addrspace(1)* %arrayidx, align 4
+; GCN: store float %tmp, float addrspace(1)* %a, align 4
+define amdgpu_kernel void @test_pow_1i(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 1.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_2f
+; GCN: %tmp = load float, float addrspace(1)* %a, align 4
+; GCN: %__pow2 = fmul fast float %tmp, %tmp
+define amdgpu_kernel void @test_pow_2f(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 2.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_2i
+; GCN: %tmp = load float, float addrspace(1)* %a, align 4
+; GCN: %__pow2 = fmul fast float %tmp, %tmp
+define amdgpu_kernel void @test_pow_2i(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 2.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_m1f
+; GCN: %tmp = load float, float addrspace(1)* %arrayidx, align 4
+; GCN: %__powrecip = fdiv fast float 1.000000e+00, %tmp
+define amdgpu_kernel void @test_pow_m1f(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float -1.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_m1i
+; GCN: %tmp = load float, float addrspace(1)* %arrayidx, align 4
+; GCN: %__powrecip = fdiv fast float 1.000000e+00, %tmp
+define amdgpu_kernel void @test_pow_m1i(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float -1.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_half
+; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float 5.000000e-01)
+; GCN-PRELINK: %__pow2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
+define amdgpu_kernel void @test_pow_half(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 5.000000e-01)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_mhalf
+; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float -5.000000e-01)
+; GCN-PRELINK: %__pow2rsqrt = tail call fast float @_Z5rsqrtf(float %tmp)
+define amdgpu_kernel void @test_pow_mhalf(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float -5.000000e-01)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow_c
+; GCN: %__powx2 = fmul fast float %tmp, %tmp
+; GCN: %__powx21 = fmul fast float %__powx2, %__powx2
+; GCN: %__powx22 = fmul fast float %__powx2, %tmp
+; GCN: %[[r0:.*]] = fmul fast float %__powx21, %__powx21
+; GCN: %__powprod3 = fmul fast float %[[r0]], %__powx22
+define amdgpu_kernel void @test_pow_c(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 1.100000e+01)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_powr_c
+; GCN: %__powx2 = fmul fast float %tmp, %tmp
+; GCN: %__powx21 = fmul fast float %__powx2, %__powx2
+; GCN: %__powx22 = fmul fast float %__powx2, %tmp
+; GCN: %[[r0:.*]] = fmul fast float %__powx21, %__powx21
+; GCN: %__powprod3 = fmul fast float %[[r0]], %__powx22
+define amdgpu_kernel void @test_powr_c(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z4powrff(float %tmp, float 1.100000e+01)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z4powrff(float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pown_c
+; GCN: %__powx2 = fmul fast float %tmp, %tmp
+; GCN: %__powx21 = fmul fast float %__powx2, %__powx2
+; GCN: %__powx22 = fmul fast float %__powx2, %tmp
+; GCN: %[[r0:.*]] = fmul fast float %__powx21, %__powx21
+; GCN: %__powprod3 = fmul fast float %[[r0]], %__powx22
+define amdgpu_kernel void @test_pown_c(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z4pownfi(float %tmp, i32 11)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z4pownfi(float, i32)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pow
+; GCN-POSTLINK: tail call fast float @_Z3powff(float %tmp, float 1.013000e+03)
+; GCN-PRELINK: %__fabs = tail call fast float @_Z4fabsf(float %tmp)
+; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %__fabs)
+; GCN-PRELINK: %__ylogx = fmul fast float %__log2, 1.013000e+03
+; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: %[[r0:.*]] = bitcast float %tmp to i32
+; GCN-PRELINK: %__pow_sign = and i32 %[[r0]], -2147483648
+; GCN-PRELINK: %[[r1:.*]] = bitcast float %__exp2 to i32
+; GCN-PRELINK: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN-PRELINK: %[[r3:.*]] = bitcast float addrspace(1)* %a to i32 addrspace(1)*
+; GCN-PRELINK: store i32 %[[r2]], i32 addrspace(1)* %[[r3]], align 4
+define amdgpu_kernel void @test_pow(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3powff(float %tmp, float 1.013000e+03)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_powr
+; GCN-POSTLINK: tail call fast float @_Z4powrff(float %tmp, float %tmp1)
+; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %tmp)
+; GCN-PRELINK: %__ylogx = fmul fast float %__log2, %tmp1
+; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: store float %__exp2, float addrspace(1)* %a, align 4
+; GCN-NATIVE:  %__log2 = tail call fast float @_Z11native_log2f(float %tmp)
+; GCN-NATIVE:  %__ylogx = fmul fast float %__log2, %tmp1
+; GCN-NATIVE:  %__exp2 = tail call fast float @_Z11native_exp2f(float %__ylogx)
+; GCN-NATIVE:  store float %__exp2, float addrspace(1)* %a, align 4
+define amdgpu_kernel void @test_powr(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
+  %call = tail call fast float @_Z4powrff(float %tmp, float %tmp1)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pown
+; GCN-POSTLINK: tail call fast float @_Z4pownfi(float %tmp, i32 %conv)
+; GCN-PRELINK: %conv = fptosi float %tmp1 to i32
+; GCN-PRELINK: %__fabs = tail call fast float @_Z4fabsf(float %tmp)
+; GCN-PRELINK: %__log2 = tail call fast float @_Z4log2f(float %__fabs)
+; GCN-PRELINK: %pownI2F = sitofp i32 %conv to float
+; GCN-PRELINK: %__ylogx = fmul fast float %__log2, %pownI2F
+; GCN-PRELINK: %__exp2 = tail call fast float @_Z4exp2f(float %__ylogx)
+; GCN-PRELINK: %__yeven = shl i32 %conv, 31
+; GCN-PRELINK: %[[r0:.*]] = bitcast float %tmp to i32
+; GCN-PRELINK: %__pow_sign = and i32 %__yeven, %[[r0]]
+; GCN-PRELINK: %[[r1:.*]] = bitcast float %__exp2 to i32
+; GCN-PRELINK: %[[r2:.*]] = or i32 %__pow_sign, %[[r1]]
+; GCN-PRELINK: %[[r3:.*]] = bitcast float addrspace(1)* %a to i32 addrspace(1)*
+; GCN-PRELINK: store i32 %[[r2]], i32 addrspace(1)* %[[r3]], align 4
+define amdgpu_kernel void @test_pown(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
+  %conv = fptosi float %tmp1 to i32
+  %call = tail call fast float @_Z4pownfi(float %tmp, i32 %conv)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_1
+; GCN: %tmp = load float, float addrspace(1)* %arrayidx, align 4
+; GCN: store float %tmp, float addrspace(1)* %a, align 4
+define amdgpu_kernel void @test_rootn_1(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 1)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z5rootnfi(float, i32)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_2
+; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 2)
+; GCN-PRELINK: %__rootn2sqrt = tail call fast float @_Z4sqrtf(float %tmp)
+define amdgpu_kernel void @test_rootn_2(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 2)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_3
+; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 3)
+; GCN-PRELINK: %__rootn2cbrt = tail call fast float @_Z4cbrtf(float %tmp)
+define amdgpu_kernel void @test_rootn_3(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 3)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_m1
+; GCN: fdiv fast float 1.000000e+00, %tmp
+define amdgpu_kernel void @test_rootn_m1(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 -1)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_rootn_m2
+; GCN-POSTLINK: tail call fast float @_Z5rootnfi(float %tmp, i32 -2)
+; GCN-PRELINK: %__rootn2rsqrt = tail call fast float @_Z5rsqrtf(float %tmp)
+define amdgpu_kernel void @test_rootn_m2(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5rootnfi(float %tmp, i32 -2)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_fma_0x
+; GCN: store float %y, float addrspace(1)* %a
+define amdgpu_kernel void @test_fma_0x(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3fmafff(float 0.000000e+00, float %tmp, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3fmafff(float, float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_fma_x0
+; GCN: store float %y, float addrspace(1)* %a
+define amdgpu_kernel void @test_fma_x0(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3fmafff(float %tmp, float 0.000000e+00, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_mad_0x
+; GCN: store float %y, float addrspace(1)* %a
+define amdgpu_kernel void @test_mad_0x(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3madfff(float 0.000000e+00, float %tmp, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3madfff(float, float, float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_mad_x0
+; GCN: store float %y, float addrspace(1)* %a
+define amdgpu_kernel void @test_mad_x0(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3madfff(float %tmp, float 0.000000e+00, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_fma_x1y
+; GCN: %fmaadd = fadd fast float %tmp, %y
+define amdgpu_kernel void @test_fma_x1y(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3fmafff(float %tmp, float 1.000000e+00, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_fma_1xy
+; GCN: %fmaadd = fadd fast float %tmp, %y
+define amdgpu_kernel void @test_fma_1xy(float addrspace(1)* nocapture %a, float %y) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3fmafff(float 1.000000e+00, float %tmp, float %y)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_fma_xy0
+; GCN: %fmamul = fmul fast float %tmp1, %tmp
+define amdgpu_kernel void @test_fma_xy0(float addrspace(1)* nocapture %a) {
+entry:
+  %arrayidx = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp = load float, float addrspace(1)* %arrayidx, align 4
+  %tmp1 = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3fmafff(float %tmp, float %tmp1, float 0.000000e+00)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp
+; GCN-NATIVE: tail call fast float @_Z10native_expf(float %tmp)
+define amdgpu_kernel void @test_use_native_exp(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3expf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3expf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp2
+; GCN-NATIVE: tail call fast float @_Z11native_exp2f(float %tmp)
+define amdgpu_kernel void @test_use_native_exp2(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z4exp2f(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z4exp2f(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_exp10
+; GCN-NATIVE: tail call fast float @_Z12native_exp10f(float %tmp)
+define amdgpu_kernel void @test_use_native_exp10(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5exp10f(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z5exp10f(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log
+; GCN-NATIVE: tail call fast float @_Z10native_logf(float %tmp)
+define amdgpu_kernel void @test_use_native_log(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3logf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3logf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log2
+; GCN-NATIVE: tail call fast float @_Z11native_log2f(float %tmp)
+define amdgpu_kernel void @test_use_native_log2(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z4log2f(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z4log2f(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_log10
+; GCN-NATIVE: tail call fast float @_Z12native_log10f(float %tmp)
+define amdgpu_kernel void @test_use_native_log10(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5log10f(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z5log10f(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_powr
+; GCN-NATIVE: %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
+; GCN-NATIVE: %__log2 = tail call fast float @_Z11native_log2f(float %tmp)
+; GCN-NATIVE: %__ylogx = fmul fast float %__log2, %tmp1
+; GCN-NATIVE: %__exp2 = tail call fast float @_Z11native_exp2f(float %__ylogx)
+; GCN-NATIVE: store float %__exp2, float addrspace(1)* %a, align 4
+define amdgpu_kernel void @test_use_native_powr(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp1 = load float, float addrspace(1)* %arrayidx1, align 4
+  %call = tail call fast float @_Z4powrff(float %tmp, float %tmp1)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_sqrt
+; GCN-NATIVE: tail call fast float @_Z11native_sqrtf(float %tmp)
+define amdgpu_kernel void @test_use_native_sqrt(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z4sqrtf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_dont_use_native_sqrt_fast_f64
+; GCN: tail call fast double @_Z4sqrtd(double %tmp)
+define amdgpu_kernel void @test_dont_use_native_sqrt_fast_f64(double addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load double, double addrspace(1)* %a, align 8
+  %call = tail call fast double @_Z4sqrtd(double %tmp)
+  store double %call, double addrspace(1)* %a, align 8
+  ret void
+}
+
+declare float @_Z4sqrtf(float)
+declare double @_Z4sqrtd(double)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_rsqrt
+; GCN-NATIVE: tail call fast float @_Z12native_rsqrtf(float %tmp)
+define amdgpu_kernel void @test_use_native_rsqrt(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z5rsqrtf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z5rsqrtf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_tan
+; GCN-NATIVE: tail call fast float @_Z10native_tanf(float %tmp)
+define amdgpu_kernel void @test_use_native_tan(float addrspace(1)* nocapture %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %call = tail call fast float @_Z3tanf(float %tmp)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z3tanf(float)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_use_native_sincos
+; GCN-NATIVE: tail call float @_Z10native_sinf(float %tmp)
+; GCN-NATIVE: tail call float @_Z10native_cosf(float %tmp)
+define amdgpu_kernel void @test_use_native_sincos(float addrspace(1)* %a) {
+entry:
+  %tmp = load float, float addrspace(1)* %a, align 4
+  %arrayidx1 = getelementptr inbounds float, float addrspace(1)* %a, i64 1
+  %tmp1 = addrspacecast float addrspace(1)* %arrayidx1 to float*
+  %call = tail call fast float @_Z6sincosfPf(float %tmp, float* %tmp1)
+  store float %call, float addrspace(1)* %a, align 4
+  ret void
+}
+
+declare float @_Z6sincosfPf(float, float*)
+
+%opencl.pipe_t = type opaque
+%opencl.reserve_id_t = type opaque
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 addrspace(1)* %ptr)
+; GCN-PRELINK: call i32 @__read_pipe_2_4(%opencl.pipe_t addrspace(1)* %{{.*}}, i32* %{{.*}}) #[[$NOUNWIND:[0-9]+]]
+; GCN-PRELINK: call i32 @__read_pipe_4_4(%opencl.pipe_t addrspace(1)* %{{.*}}, %opencl.reserve_id_t addrspace(5)* %{{.*}}, i32 2, i32* %{{.*}}) #[[$NOUNWIND]]
+define amdgpu_kernel void @test_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 addrspace(1)* %ptr) local_unnamed_addr {
+entry:
+  %tmp = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
+  %tmp1 = addrspacecast i8 addrspace(1)* %tmp to i8*
+  %tmp2 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
+  %tmp3 = tail call %opencl.reserve_id_t addrspace(5)* @__reserve_read_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4)
+  %tmp4 = tail call i32 @__read_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
+  tail call void @__commit_read_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4)
+  ret void
+}
+
+declare i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)*, i8*, i32, i32)
+
+declare %opencl.reserve_id_t addrspace(5)* @__reserve_read_pipe(%opencl.pipe_t addrspace(1)*, i32, i32, i32)
+
+declare i32 @__read_pipe_4(%opencl.pipe_t addrspace(1)*, %opencl.reserve_id_t addrspace(5)*, i32, i8*, i32, i32)
+
+declare void @__commit_read_pipe(%opencl.pipe_t addrspace(1)*, %opencl.reserve_id_t addrspace(5)*, i32, i32)
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32 addrspace(1)* %ptr)
+; GCN-PRELINK: call i32 @__write_pipe_2_4(%opencl.pipe_t addrspace(1)* %{{.*}}, i32* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__write_pipe_4_4(%opencl.pipe_t addrspace(1)* %{{.*}}, %opencl.reserve_id_t addrspace(5)* %{{.*}}, i32 2, i32* %{{.*}}) #[[$NOUNWIND]]
+define amdgpu_kernel void @test_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32 addrspace(1)* %ptr) local_unnamed_addr {
+entry:
+  %tmp = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
+  %tmp1 = addrspacecast i8 addrspace(1)* %tmp to i8*
+  %tmp2 = tail call i32 @__write_pipe_2(%opencl.pipe_t addrspace(1)* %p, i8* %tmp1, i32 4, i32 4) #0
+  %tmp3 = tail call %opencl.reserve_id_t addrspace(5)* @__reserve_write_pipe(%opencl.pipe_t addrspace(1)* %p, i32 2, i32 4, i32 4) #0
+  %tmp4 = tail call i32 @__write_pipe_4(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 2, i8* %tmp1, i32 4, i32 4) #0
+  tail call void @__commit_write_pipe(%opencl.pipe_t addrspace(1)* %p, %opencl.reserve_id_t addrspace(5)* %tmp3, i32 4, i32 4) #0
+  ret void
+}
+
+declare i32 @__write_pipe_2(%opencl.pipe_t addrspace(1)*, i8*, i32, i32) local_unnamed_addr
+
+declare %opencl.reserve_id_t addrspace(5)* @__reserve_write_pipe(%opencl.pipe_t addrspace(1)*, i32, i32, i32) local_unnamed_addr
+
+declare i32 @__write_pipe_4(%opencl.pipe_t addrspace(1)*, %opencl.reserve_id_t addrspace(5)*, i32, i8*, i32, i32) local_unnamed_addr
+
+declare void @__commit_write_pipe(%opencl.pipe_t addrspace(1)*, %opencl.reserve_id_t addrspace(5)*, i32, i32) local_unnamed_addr
+
+%struct.S = type { [100 x i32] }
+
+; GCN-LABEL: {{^}}define amdgpu_kernel void @test_pipe_size
+; GCN-PRELINK: call i32 @__read_pipe_2_1(%opencl.pipe_t addrspace(1)* %{{.*}} i8* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_2(%opencl.pipe_t addrspace(1)* %{{.*}} i16* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_4(%opencl.pipe_t addrspace(1)* %{{.*}} i32* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_8(%opencl.pipe_t addrspace(1)* %{{.*}} i64* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_16(%opencl.pipe_t addrspace(1)* %{{.*}}, <2 x i64>* %{{.*}}) #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_32(%opencl.pipe_t addrspace(1)* %{{.*}}, <4 x i64>* %{{.*}} #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_64(%opencl.pipe_t addrspace(1)* %{{.*}}, <8 x i64>* %{{.*}} #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2_128(%opencl.pipe_t addrspace(1)* %{{.*}}, <16 x i64>* %{{.*}} #[[$NOUNWIND]]
+; GCN-PRELINK: call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %{{.*}}, i8* %{{.*}} i32 400, i32 4) #[[$NOUNWIND]]
+define amdgpu_kernel void @test_pipe_size(%opencl.pipe_t addrspace(1)* %p1, i8 addrspace(1)* %ptr1, %opencl.pipe_t addrspace(1)* %p2, i16 addrspace(1)* %ptr2, %opencl.pipe_t addrspace(1)* %p4, i32 addrspace(1)* %ptr4, %opencl.pipe_t addrspace(1)* %p8, i64 addrspace(1)* %ptr8, %opencl.pipe_t addrspace(1)* %p16, <2 x i64> addrspace(1)* %ptr16, %opencl.pipe_t addrspace(1)* %p32, <4 x i64> addrspace(1)* %ptr32, %opencl.pipe_t addrspace(1)* %p64, <8 x i64> addrspace(1)* %ptr64, %opencl.pipe_t addrspace(1)* %p128, <16 x i64> addrspace(1)* %ptr128, %opencl.pipe_t addrspace(1)* %pu, %struct.S addrspace(1)* %ptru) local_unnamed_addr #0 {
+entry:
+  %tmp = addrspacecast i8 addrspace(1)* %ptr1 to i8*
+  %tmp1 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p1, i8* %tmp, i32 1, i32 1) #0
+  %tmp2 = bitcast i16 addrspace(1)* %ptr2 to i8 addrspace(1)*
+  %tmp3 = addrspacecast i8 addrspace(1)* %tmp2 to i8*
+  %tmp4 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p2, i8* %tmp3, i32 2, i32 2) #0
+  %tmp5 = bitcast i32 addrspace(1)* %ptr4 to i8 addrspace(1)*
+  %tmp6 = addrspacecast i8 addrspace(1)* %tmp5 to i8*
+  %tmp7 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p4, i8* %tmp6, i32 4, i32 4) #0
+  %tmp8 = bitcast i64 addrspace(1)* %ptr8 to i8 addrspace(1)*
+  %tmp9 = addrspacecast i8 addrspace(1)* %tmp8 to i8*
+  %tmp10 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p8, i8* %tmp9, i32 8, i32 8) #0
+  %tmp11 = bitcast <2 x i64> addrspace(1)* %ptr16 to i8 addrspace(1)*
+  %tmp12 = addrspacecast i8 addrspace(1)* %tmp11 to i8*
+  %tmp13 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p16, i8* %tmp12, i32 16, i32 16) #0
+  %tmp14 = bitcast <4 x i64> addrspace(1)* %ptr32 to i8 addrspace(1)*
+  %tmp15 = addrspacecast i8 addrspace(1)* %tmp14 to i8*
+  %tmp16 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p32, i8* %tmp15, i32 32, i32 32) #0
+  %tmp17 = bitcast <8 x i64> addrspace(1)* %ptr64 to i8 addrspace(1)*
+  %tmp18 = addrspacecast i8 addrspace(1)* %tmp17 to i8*
+  %tmp19 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p64, i8* %tmp18, i32 64, i32 64) #0
+  %tmp20 = bitcast <16 x i64> addrspace(1)* %ptr128 to i8 addrspace(1)*
+  %tmp21 = addrspacecast i8 addrspace(1)* %tmp20 to i8*
+  %tmp22 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %p128, i8* %tmp21, i32 128, i32 128) #0
+  %tmp23 = bitcast %struct.S addrspace(1)* %ptru to i8 addrspace(1)*
+  %tmp24 = addrspacecast i8 addrspace(1)* %tmp23 to i8*
+  %tmp25 = tail call i32 @__read_pipe_2(%opencl.pipe_t addrspace(1)* %pu, i8* %tmp24, i32 400, i32 4) #0
+  ret void
+}
+
+; CGN-PRELINK: attributes #[[$NOUNWIND]] = { nounwind }
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll b/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
new file mode 100644
index 0000000000000..f8077cd8e3ab0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/simplifydemandedbits-recursion.ll
@@ -0,0 +1,95 @@
+; RUN: llc -march=amdgcn < %s | FileCheck %s
+
+; Check we can compile this bugpoint-reduced test without an
+; infinite loop in TLI.SimplifyDemandedBits() due to failure
+; to use return value of TLO.DAG.UpdateNodeOperands()
+
+; Check that code was generated; we know there will be
+; a s_endpgm, so check for it.
+
+@0 = external unnamed_addr addrspace(3) global [462 x float], align 4
+
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.workitem.id.y() #0
+
+; Function Attrs: nounwind readnone speculatable
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+; Function Attrs: nounwind readnone speculatable
+declare float @llvm.fmuladd.f32(float, float, float) #0
+
+; CHECK: s_endpgm
+define amdgpu_kernel void @foo(float addrspace(1)* noalias nocapture readonly %arg, float addrspace(1)* noalias nocapture readonly %arg1, float addrspace(1)* noalias nocapture %arg2, float %arg3) local_unnamed_addr !reqd_work_group_size !0 {
+bb:
+  %tmp = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %tmp4 = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp5 = and i32 %tmp, 15
+  %tmp6 = mul nuw nsw i32 %tmp5, 21
+  %tmp7 = sub i32 %tmp6, 0
+  %tmp8 = add i32 %tmp7, 0
+  %tmp9 = add i32 %tmp8, 0
+  %tmp10 = getelementptr inbounds [462 x float], [462 x float] addrspace(3)* @0, i32 0, i32 0
+  br label %bb12
+
+bb11:                                             ; preds = %bb30
+  br i1 undef, label %bb37, label %bb38
+
+bb12:                                             ; preds = %bb30, %bb
+  br i1 false, label %.preheader, label %.loopexit145
+
+.loopexit145:                                     ; preds = %.preheader, %bb12
+  br label %bb13
+
+bb13:                                             ; preds = %.loopexit, %.loopexit145
+  %tmp14 = phi i32 [ %tmp5, %.loopexit145 ], [ %tmp20, %.loopexit ]
+  %tmp15 = add nsw i32 %tmp14, -3
+  %tmp16 = mul i32 %tmp14, 21
+  br i1 undef, label %bb17, label %.loopexit
+
+bb17:                                             ; preds = %bb13
+  %tmp18 = mul i32 %tmp15, 224
+  %tmp19 = add i32 undef, %tmp18
+  br label %bb21
+
+.loopexit:                                        ; preds = %bb21, %bb13
+  %tmp20 = add nuw nsw i32 %tmp14, 16
+  br i1 undef, label %bb13, label %bb26
+
+bb21:                                             ; preds = %bb21, %bb17
+  %tmp22 = phi i32 [ %tmp4, %bb17 ], [ %tmp25, %bb21 ]
+  %tmp23 = add i32 %tmp22, %tmp16
+  %tmp24 = getelementptr inbounds float, float addrspace(3)* %tmp10, i32 %tmp23
+  store float undef, float addrspace(3)* %tmp24, align 4
+  %tmp25 = add nuw i32 %tmp22, 8
+  br i1 undef, label %bb21, label %.loopexit
+
+bb26:                                             ; preds = %.loopexit
+  br label %bb31
+
+.preheader:                                       ; preds = %.preheader, %bb12
+  %tmp27 = phi i32 [ %tmp28, %.preheader ], [ undef, %bb12 ]
+  %tmp28 = add nuw i32 %tmp27, 128
+  %tmp29 = icmp ult i32 %tmp28, 1568
+  br i1 %tmp29, label %.preheader, label %.loopexit145
+
+bb30:                                             ; preds = %bb31
+  br i1 undef, label %bb11, label %bb12
+
+bb31:                                             ; preds = %bb31, %bb26
+  %tmp32 = phi i32 [ %tmp9, %bb26 ], [ undef, %bb31 ]
+  %tmp33 = getelementptr inbounds [462 x float], [462 x float] addrspace(3)* @0, i32 0, i32 %tmp32
+  %tmp34 = load float, float addrspace(3)* %tmp33, align 4
+  %tmp35 = tail call float @llvm.fmuladd.f32(float %tmp34, float undef, float undef)
+  %tmp36 = tail call float @llvm.fmuladd.f32(float undef, float undef, float %tmp35)
+  br i1 undef, label %bb30, label %bb31
+
+bb37:                                             ; preds = %bb11
+  br label %bb38
+
+bb38:                                             ; preds = %bb37, %bb11
+  ret void
+}
+
+attributes #0 = { nounwind readnone speculatable }
+
+!0 = !{i32 8, i32 16, i32 1}
diff --git a/test/CodeGen/AMDGPU/sitofp.f16.ll b/test/CodeGen/AMDGPU/sitofp.f16.ll
index 0bcef99df39f6..31d34663489bf 100644
--- a/test/CodeGen/AMDGPU/sitofp.f16.ll
+++ b/test/CodeGen/AMDGPU/sitofp.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}sitofp_i16_to_f16
 ; GCN: buffer_load_{{sshort|ushort}} v[[A_I16:[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/skip-if-dead.ll b/test/CodeGen/AMDGPU/skip-if-dead.ll
index ea8b87f1dee23..9ae36b0a06c21 100644
--- a/test/CodeGen/AMDGPU/skip-if-dead.ll
+++ b/test/CodeGen/AMDGPU/skip-if-dead.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s
 
 ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_pos:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_0_imm_pos() #0 {
   call void @llvm.AMDGPU.kill(float 0.0)
@@ -9,9 +9,9 @@ define amdgpu_ps void @test_kill_depth_0_imm_pos() #0 {
 }
 
 ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_0_imm_neg() #0 {
   call void @llvm.AMDGPU.kill(float -0.0)
@@ -20,11 +20,11 @@ define amdgpu_ps void @test_kill_depth_0_imm_neg() #0 {
 
 ; FIXME: Ideally only one would be emitted
 ; CHECK-LABEL: {{^}}test_kill_depth_0_imm_neg_x2:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: s_mov_b64 exec, 0
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_0_imm_neg_x2() #0 {
   call void @llvm.AMDGPU.kill(float -0.0)
@@ -33,9 +33,9 @@ define amdgpu_ps void @test_kill_depth_0_imm_neg_x2() #0 {
 }
 
 ; CHECK-LABEL: {{^}}test_kill_depth_var:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var(float %x) #0 {
   call void @llvm.AMDGPU.kill(float %x)
@@ -44,11 +44,11 @@ define amdgpu_ps void @test_kill_depth_var(float %x) #0 {
 
 ; FIXME: Ideally only one would be emitted
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_same:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v0
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2_same(float %x) #0 {
   call void @llvm.AMDGPU.kill(float %x)
@@ -57,11 +57,11 @@ define amdgpu_ps void @test_kill_depth_var_x2_same(float %x) #0 {
 }
 
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v1
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2(float %x, float %y) #0 {
   call void @llvm.AMDGPU.kill(float %x)
@@ -70,12 +70,12 @@ define amdgpu_ps void @test_kill_depth_var_x2(float %x, float %y) #0 {
 }
 
 ; CHECK-LABEL: {{^}}test_kill_depth_var_x2_instructions:
-; CHECK-NEXT: ; BB#0:
+; CHECK-NEXT: ; %bb.0:
 ; CHECK-NEXT: v_cmpx_le_f32_e32 vcc, 0, v0
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK: v_mov_b32_e64 v7, -1
 ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: s_endpgm
 define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 {
   call void @llvm.AMDGPU.kill(float %x)
@@ -90,7 +90,7 @@ define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 {
 ; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0
 ; CHECK: s_cbranch_scc1 [[RETURN_BB:BB[0-9]+_[0-9]+]]
 
-; CHECK-NEXT: ; BB#1:
+; CHECK-NEXT: ; %bb.1:
 ; CHECK: v_mov_b32_e64 v7, -1
 ; CHECK: v_nop_e64
 ; CHECK: v_nop_e64
@@ -105,7 +105,7 @@ define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 {
 
 ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7
 ; CHECK-NEXT: s_cbranch_execnz [[SPLIT_BB:BB[0-9]+_[0-9]+]]
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
 
@@ -141,7 +141,7 @@ exit:
 ; CHECK-NEXT: v_mov_b32_e32 v{{[0-9]+}}, 0
 ; CHECK-NEXT: s_cbranch_scc1 [[RETURN_BB:BB[0-9]+_[0-9]+]]
 
-; CHECK-NEXT: ; BB#1: ; %bb
+; CHECK-NEXT: ; %bb.1: ; %bb
 ; CHECK: v_mov_b32_e64 v7, -1
 ; CHECK: v_nop_e64
 ; CHECK: v_nop_e64
@@ -157,7 +157,7 @@ exit:
 ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7
 ; CHECK-NEXT: s_cbranch_execnz [[SPLIT_BB:BB[0-9]+_[0-9]+]]
 
-; CHECK-NEXT: ; BB#2:
+; CHECK-NEXT: ; %bb.2:
 ; CHECK-NEXT: exp null off, off, off, off done vm
 ; CHECK-NEXT: s_endpgm
 
@@ -215,7 +215,7 @@ exit:
 ; CHECK: v_nop_e64
 ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7
 
-; CHECK-NEXT: ; BB#3:
+; CHECK-NEXT: ; %bb.3:
 ; CHECK: buffer_load_dword [[LOAD:v[0-9]+]]
 ; CHECK: v_cmp_eq_u32_e32 vcc, 0, [[LOAD]]
 ; CHECK-NEXT: s_and_b64 vcc, exec, vcc
@@ -309,7 +309,7 @@ end:
 
 ; CHECK: [[SKIPKILL]]:
 ; CHECK: v_cmp_nge_f32_e32 vcc
-; CHECK-NEXT: BB#3: ; %bb5
+; CHECK-NEXT: %bb.3: ; %bb5
 ; CHECK-NEXT: .Lfunc_end{{[0-9]+}}
 define amdgpu_ps void @no_skip_no_successors(float inreg %arg, float inreg %arg1) #0 {
 bb:
@@ -335,7 +335,7 @@ bb7:                                              ; preds = %bb4
 }
 
 ; CHECK-LABEL: {{^}}if_after_kill_block:
-; CHECK: ; BB#0:
+; CHECK: ; %bb.0:
 ; CHECK: s_and_saveexec_b64
 ; CHECK: s_xor_b64
 ; CHECK-NEXT: mask branch [[BB4:BB[0-9]+_[0-9]+]]
@@ -347,7 +347,6 @@ bb7:                                              ; preds = %bb4
 
 ; CHECK: v_cmp_neq_f32_e32 vcc, 0,
 ; CHECK: s_and_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, vcc
-; CHECK: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
 ; CHECK: mask branch [[END:BB[0-9]+_[0-9]+]]
 ; CHECK-NOT: branch
 
@@ -355,7 +354,6 @@ bb7:                                              ; preds = %bb4
 ; CHECK: buffer_store_dword
 
 ; CHECK: [[END]]:
-; CHECK: s_or_b64 exec, exec
 ; CHECK: s_endpgm
 define amdgpu_ps void @if_after_kill_block(float %arg, float %arg1, <4 x float> %arg2) #0 {
 bb:
diff --git a/test/CodeGen/AMDGPU/smed3.ll b/test/CodeGen/AMDGPU/smed3.ll
index 8665ab697265e..9e10f049c607e 100644
--- a/test/CodeGen/AMDGPU/smed3.ll
+++ b/test/CodeGen/AMDGPU/smed3.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SICIVI -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
diff --git a/test/CodeGen/AMDGPU/sminmax.ll b/test/CodeGen/AMDGPU/sminmax.ll
index 41430715f3476..96a318fef024b 100644
--- a/test/CodeGen/AMDGPU/sminmax.ll
+++ b/test/CodeGen/AMDGPU/sminmax.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_abs_i32:
 ; GCN: s_abs_i32
@@ -17,9 +18,13 @@ define amdgpu_kernel void @s_abs_i32(i32 addrspace(1)* %out, i32 %val) nounwind
 }
 
 ; FUNC-LABEL: {{^}}v_abs_i32:
-; GCN: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
+; SIVI: v_sub_{{i|u}}32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
+; GFX9: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SRC:v[0-9]+]]
+
 ; GCN: v_max_i32_e32 {{v[0-9]+}}, [[SRC]], [[NEG]]
-; GCN: v_add_i32
+
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
@@ -33,7 +38,8 @@ define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %
 }
 
 ; GCN-LABEL: {{^}}v_abs_i32_repeat_user:
-; GCN: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
+; SIVI: v_sub_{{i|u}}32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
+; GFX9: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SRC:v[0-9]+]]
 ; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[SRC]], [[NEG]]
 ; GCN: v_mul_lo_i32 v{{[0-9]+}}, [[MAX]], [[MAX]]
 define amdgpu_kernel void @v_abs_i32_repeat_user(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
@@ -68,14 +74,20 @@ define amdgpu_kernel void @s_abs_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %
 }
 
 ; FUNC-LABEL: {{^}}v_abs_v2i32:
-; GCN-DAG: v_sub_i32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
-; GCN-DAG: v_sub_i32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
+
+; GFX9-DAG: v_sub_u32_e32 [[NEG0:v[0-9]+]], 0, [[SRC0:v[0-9]+]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG1:v[0-9]+]], 0, [[SRC1:v[0-9]+]]
 
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC0]], [[NEG0]]
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC1]], [[NEG1]]
 
-; GCN: v_add_i32
-; GCN: v_add_i32
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
+
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
 
 ; EG: MAX_INT
 ; EG: MAX_INT
@@ -127,20 +139,31 @@ define amdgpu_kernel void @s_abs_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %
 }
 
 ; FUNC-LABEL: {{^}}v_abs_v4i32:
-; GCN-DAG: v_sub_i32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
-; GCN-DAG: v_sub_i32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
-; GCN-DAG: v_sub_i32_e32 [[NEG2:v[0-9]+]], vcc, 0, [[SRC2:v[0-9]+]]
-; GCN-DAG: v_sub_i32_e32 [[NEG3:v[0-9]+]], vcc, 0, [[SRC3:v[0-9]+]]
+
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG2:v[0-9]+]], vcc, 0, [[SRC2:v[0-9]+]]
+; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG3:v[0-9]+]], vcc, 0, [[SRC3:v[0-9]+]]
+
+; GFX9-DAG: v_sub_u32_e32 [[NEG0:v[0-9]+]], 0, [[SRC0:v[0-9]+]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG1:v[0-9]+]], 0, [[SRC1:v[0-9]+]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG2:v[0-9]+]], 0, [[SRC2:v[0-9]+]]
+; GFX9-DAG: v_sub_u32_e32 [[NEG3:v[0-9]+]], 0, [[SRC3:v[0-9]+]]
 
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC0]], [[NEG0]]
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC1]], [[NEG1]]
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC2]], [[NEG2]]
 ; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC3]], [[NEG3]]
 
-; GCN: v_add_i32
-; GCN: v_add_i32
-; GCN: v_add_i32
-; GCN: v_add_i32
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
+; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
+
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
+; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
 
 ; EG: MAX_INT
 ; EG: MAX_INT
@@ -181,8 +204,8 @@ define amdgpu_kernel void @s_min_max_i32(i32 addrspace(1)* %out0, i32 addrspace(
 }
 
 ; FUNC-LABEL: {{^}}v_min_max_i32:
-; GCN: {{buffer|flat}}_load_dword [[VAL0:v[0-9]+]]
-; GCN: {{buffer|flat}}_load_dword [[VAL1:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[VAL0:v[0-9]+]]
+; GCN: {{buffer|flat|global}}_load_dword [[VAL1:v[0-9]+]]
 
 ; GCN-DAG: v_min_i32_e32 v{{[0-9]+}}, [[VAL0]], [[VAL1]]
 ; GCN-DAG: v_max_i32_e32 v{{[0-9]+}}, [[VAL0]], [[VAL1]]
diff --git a/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 27263429650d8..ffad4f6a4969b 100644
--- a/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=CIVI -check-prefix=GCN %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=CIVI -check-prefix=GCN %s
 
@@ -8,12 +8,12 @@
 ; GFX9: v_pk_max_i16 [[MAX:v[0-9]+]], [[VAL]], [[SUB]]
 ; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]], [[MAX]], 2
 
-; VI: v_sub_i32_e32
-; VI-DAG: v_sub_i32_e32
-; VI: v_max_i32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, sext(v{{[0-9]+}}) dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; VI: v_max_i32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; VI: v_add_i32_e32
-; VI: v_add_i32_e32
+; VI: v_sub_u32_e32
+; VI-DAG: v_sub_u32_e32
+; VI: v_max_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; VI: v_max_i32_sdwa v{{[0-9]+}}, sext(v{{[0-9]+}}), sext(v{{[0-9]+}}) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; VI: v_add_u32_e32
+; VI: v_add_u32_e32
 ; VI: v_or_b32_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
 
 ; CI: v_sub_i32_e32
@@ -35,7 +35,7 @@ define amdgpu_kernel void @s_abs_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %
 }
 
 ; GCN-LABEL: {{^}}v_abs_v2i16:
-; GFX9: flat_load_dword [[VAL:v[0-9]+]]
+; GFX9: global_load_dword [[VAL:v[0-9]+]]
 ; GFX9: v_pk_sub_i16 [[SUB:v[0-9]+]], 0, [[VAL]]
 ; GFX9: v_pk_max_i16 [[MAX:v[0-9]+]], [[VAL]], [[SUB]]
 ; GFX9: v_pk_add_u16 [[ADD:v[0-9]+]], [[MAX]], 2
diff --git a/test/CodeGen/AMDGPU/smrd.ll b/test/CodeGen/AMDGPU/smrd.ll
index 3f1e1cacb879d..9fd20fd67b8ce 100644
--- a/test/CodeGen/AMDGPU/smrd.ll
+++ b/test/CodeGen/AMDGPU/smrd.ll
@@ -1,11 +1,12 @@
-; RUN: llc -march=amdgcn -show-mc-encoding -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=SIVI %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -show-mc-encoding -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN  %s
-; RUN: llc -march=amdgcn -mcpu=tonga -show-mc-encoding -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=SIVI %s
+; RUN: llc -march=amdgcn -mcpu=tahiti  -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=SI   -check-prefix=GCN -check-prefix=SICI -check-prefix=SIVIGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=CI   -check-prefix=GCN -check-prefix=SICI %s
+; RUN: llc -march=amdgcn -mcpu=tonga   -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=VI   -check-prefix=GCN -check-prefix=VIGFX9 -check-prefix=SIVIGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900  -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN -check-prefix=VIGFX9 -check-prefix=SIVIGFX9  %s
 
 ; SMRD load with an immediate offset.
 ; GCN-LABEL: {{^}}smrd0:
 ; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x1 ; encoding: [0x01
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4
+; VIGFX9: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4
 define amdgpu_kernel void @smrd0(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(2)* %ptr, i64 1
@@ -17,7 +18,7 @@ entry:
 ; SMRD load with the largest possible immediate offset.
 ; GCN-LABEL: {{^}}smrd1:
 ; SICI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff,0x{{[0-9]+[137]}}
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
+; VIGFX9: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
 define amdgpu_kernel void @smrd1(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(2)* %ptr, i64 255
@@ -31,7 +32,7 @@ entry:
 ; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400
 ; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
+; VIGFX9: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
 ; GCN: s_endpgm
 define amdgpu_kernel void @smrd2(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) #0 {
 entry:
@@ -61,7 +62,7 @@ entry:
 ; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc
 ; SI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff
-; VI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
+; VIGFX9: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
 define amdgpu_kernel void @smrd4(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) #0 {
 entry:
   %tmp = getelementptr i32, i32 addrspace(2)* %ptr, i64 262143
@@ -72,8 +73,8 @@ entry:
 
 ; SMRD load with an offset greater than the largest possible immediate on VI
 ; GCN-LABEL: {{^}}smrd5:
-; SIVI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
-; SIVI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
+; SIVIGFX9: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
+; SIVIGFX9: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000
 ; GCN: s_endpgm
 define amdgpu_kernel void @smrd5(i32 addrspace(1)* %out, i32 addrspace(2)* %ptr) #0 {
@@ -84,10 +85,27 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}smrd_hazard:
+; GCN-DAG: s_mov_b32 s3, 3
+; GCN-DAG: s_mov_b32 s2, 2
+; GCN-DAG: s_mov_b32 s1, 1
+; GCN-DAG: s_mov_b32 s0, 0
+; SI-NEXT: nop 3
+; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
+define amdgpu_ps float @smrd_hazard(<4 x i32> inreg %desc) #0 {
+main_body:
+  %d0 = insertelement <4 x i32> undef, i32 0, i32 0
+  %d1 = insertelement <4 x i32> %d0, i32 1, i32 1
+  %d2 = insertelement <4 x i32> %d1, i32 2, i32 2
+  %d3 = insertelement <4 x i32> %d2, i32 3, i32 3
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %d3, i32 0)
+  ret float %r
+}
+
 ; SMRD load using the load.const.v4i32 intrinsic with an immediate offset
 ; GCN-LABEL: {{^}}smrd_load_const0:
 ; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x4 ; encoding: [0x04
-; VI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x10
+; VIGFX9: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x10
 define amdgpu_ps void @smrd_load_const0(<4 x i32> addrspace(2)* inreg %arg, <4 x i32> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(2)* %arg, i32 0
@@ -101,7 +119,7 @@ main_body:
 ; offset.
 ; GCN-LABEL: {{^}}smrd_load_const1:
 ; SICI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xff ; encoding: [0xff
-; VI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
+; VIGFX9: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3fc
 define amdgpu_ps void @smrd_load_const1(<4 x i32> addrspace(2)* inreg %arg, <4 x i32> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(2)* %arg, i32 0
@@ -118,7 +136,7 @@ main_body:
 ; SI: s_movk_i32 s[[OFFSET:[0-9]]], 0x400
 ; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], s[[OFFSET]] ; encoding: [0x0[[OFFSET]]
 ; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x100
-; VI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
+; VIGFX9: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x400
 define amdgpu_ps void @smrd_load_const2(<4 x i32> addrspace(2)* inreg %arg, <4 x i32> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(2)* %arg, i32 0
@@ -133,7 +151,7 @@ main_body:
 ; SI: s_mov_b32 [[OFFSET:s[0-9]+]], 0xffffc
 ; SI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x3ffff
-; VI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
+; VIGFX9: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0xffffc
 define amdgpu_ps void @smrd_load_const3(<4 x i32> addrspace(2)* inreg %arg, <4 x i32> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(2)* %arg, i32 0
@@ -145,8 +163,8 @@ main_body:
 
 ; SMRD load with an offset greater than the largest possible immediate on VI
 ; GCN-LABEL: {{^}}smrd_load_const4:
-; SIVI: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
-; SIVI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
+; SIVIGFX9: s_mov_b32 [[OFFSET:s[0-9]+]], 0x100000
+; SIVIGFX9: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], [[OFFSET]]
 ; CI: s_buffer_load_dword s{{[0-9]}}, s[{{[0-9]:[0-9]}}], 0x40000
 ; GCN: s_endpgm
 define amdgpu_ps void @smrd_load_const4(<4 x i32> addrspace(2)* inreg %arg, <4 x i32> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3, <2 x i32> %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <3 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, float %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19) #0 {
@@ -158,6 +176,103 @@ main_body:
   ret void
 }
 
+; GCN-LABEL: {{^}}smrd_sgpr_offset:
+; GCN: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4
+define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 {
+main_body:
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  ret float %r
+}
+
+; GCN-LABEL: {{^}}smrd_vgpr_offset:
+; GCN: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ;
+define amdgpu_ps float @smrd_vgpr_offset(<4 x i32> inreg %desc, i32 %offset) #0 {
+main_body:
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  ret float %r
+}
+
+; GCN-LABEL: {{^}}smrd_vgpr_offset_imm:
+; GCN-NEXT: %bb.
+
+; SICIVI-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen offset:4095 ;
+
+; GFX9-NEXT: v_add_u32_e32 [[ADD:v[0-9]+]], 0xfff, v0
+; GFX9-NEXT: buffer_load_dword v{{[0-9]}}, [[ADD]], s[0:3], 0 offen ;
+define amdgpu_ps float @smrd_vgpr_offset_imm(<4 x i32> inreg %desc, i32 %offset) #0 {
+main_body:
+  %off = add i32 %offset, 4095
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %off)
+  ret float %r
+}
+
+; GCN-LABEL: {{^}}smrd_vgpr_offset_imm_too_large:
+; GCN-NEXT: %bb.
+; GCN-NEXT: v_add_{{i|u}}32_e32 v0, {{(vcc, )?}}0x1000, v0
+; GCN-NEXT: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ;
+define amdgpu_ps float @smrd_vgpr_offset_imm_too_large(<4 x i32> inreg %desc, i32 %offset) #0 {
+main_body:
+  %off = add i32 %offset, 4096
+  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %off)
+  ret float %r
+}
+
+; GCN-LABEL: {{^}}smrd_imm_merged:
+; GCN-NEXT: %bb.
+; SICI-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1
+; SICI-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x7
+; VI-NEXT: s_buffer_load_dwordx4 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x4
+; VI-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1c
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+; GFX9-NEXT: s_buffer_load_dword s{{[0-9]}}
+define amdgpu_ps void @smrd_imm_merged(<4 x i32> inreg %desc) #0 {
+main_body:
+  %r1 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 4)
+  %r2 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 8)
+  %r3 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 12)
+  %r4 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 16)
+  %r5 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 28)
+  %r6 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 32)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}smrd_vgpr_merged:
+; GCN-NEXT: %bb.
+
+; SICIVI-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+; SICIVI-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
+
+; GFX9: buffer_load_dword
+; GFX9: buffer_load_dword
+; GFX9: buffer_load_dword
+; GFX9: buffer_load_dword
+; GFX9: buffer_load_dword
+; GFX9: buffer_load_dword
+define amdgpu_ps void @smrd_vgpr_merged(<4 x i32> inreg %desc, i32 %a) #0 {
+main_body:
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  %r1 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a1)
+  %r2 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a2)
+  %r3 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a3)
+  %r4 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a4)
+  %r5 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a5)
+  %r6 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a6)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0
+  ret void
+}
+
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
 
diff --git a/test/CodeGen/AMDGPU/spill-empty-live-interval.mir b/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
index bb58da7a59f63..1e9b6b5dd8d2d 100644
--- a/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
+++ b/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
@@ -2,18 +2,18 @@
 # https://bugs.llvm.org/show_bug.cgi?id=33620
 
 ---
-# This would assert due to the empty live interval created for %vreg9
+# This would assert due to the empty live interval created for %9
 # on the last S_NOP with an undef subreg use.
 
 # CHECK-LABEL: name: expecting_non_empty_interval
 
-# CHECK: undef %7.sub1 = V_MAC_F32_e32 0, undef %1, undef %7.sub1, implicit %exec
+# CHECK: undef %7.sub1:vreg_64 = V_MAC_F32_e32 0, undef %1:vgpr_32, undef %7.sub1, implicit %exec
 # CHECK-NEXT: SI_SPILL_V64_SAVE %7, %stack.0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr5, 0, implicit %exec :: (store 8 into %stack.0, align 4)
-# CHECK-NEXT: undef %5.sub1 = V_MOV_B32_e32 1786773504, implicit %exec
-# CHECK-NEXT: dead %2 = V_MUL_F32_e32 0, %5.sub1, implicit %exec
+# CHECK-NEXT: undef %5.sub1:vreg_64 = V_MOV_B32_e32 1786773504, implicit %exec
+# CHECK-NEXT: dead %2:vgpr_32 = V_MUL_F32_e32 0, %5.sub1, implicit %exec
 
 # CHECK: S_NOP 0, implicit %6.sub1
-# CHECK-NEXT: %8 = SI_SPILL_V64_RESTORE %stack.0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr5, 0, implicit %exec :: (load 8 from %stack.0, align 4)
+# CHECK-NEXT: %8:vreg_64 = SI_SPILL_V64_RESTORE %stack.0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr5, 0, implicit %exec :: (load 8 from %stack.0, align 4)
 # CHECK-NEXT: S_NOP 0, implicit %8.sub1
 # CHECK-NEXT: S_NOP 0, implicit undef %9.sub0
 
@@ -35,7 +35,6 @@ body:             |
     S_NOP 0, implicit %3.sub1
     S_NOP 0, implicit %0.sub1
     S_NOP 0, implicit undef %0.sub0
-    S_ENDPGM
 
 ...
 
@@ -45,12 +44,12 @@ body:             |
 # CHECK-LABEL: name: rematerialize_empty_interval_has_reference
 
 # CHECK-NOT: MOV
-# CHECK: undef %3.sub2 = V_MOV_B32_e32 1786773504, implicit %exec
+# CHECK: undef %3.sub2:vreg_128 = V_MOV_B32_e32 1786773504, implicit %exec
 
 # CHECK: bb.1:
 # CHECK-NEXT: S_NOP 0, implicit %3.sub2
 # CHECK-NEXT: S_NOP 0, implicit undef %6.sub0
-# CHECK-NEXT: undef %4.sub2 = V_MOV_B32_e32 0, implicit %exec
+# CHECK-NEXT: undef %4.sub2:vreg_128 = V_MOV_B32_e32 0, implicit %exec
 # CHECK-NEXT: S_NOP 0, implicit %4.sub2
 name: rematerialize_empty_interval_has_reference
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/spill-m0.ll b/test/CodeGen/AMDGPU/spill-m0.ll
index 1147464c1ddae..a7b522165ab89 100644
--- a/test/CodeGen/AMDGPU/spill-m0.ll
+++ b/test/CodeGen/AMDGPU/spill-m0.ll
@@ -66,13 +66,13 @@ endif:
 
 ; TOSMEM-NOT: s_m0
 ; TOSMEM: s_add_u32 m0, s7, 0x100
-; TOSMEM-NEXT: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill
+; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
 ; FIXME: RegScavenger::isRegUsed() always returns true if m0 is reserved, so we have to save and restore it
 ; FIXME-TOSMEM-NOT: m0
 
 ; FIXME-TOSMEM-NOT: m0
-; TOSMEM: s_add_u32 m0, s7, 0x200
-; TOSMEM: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill
+; TOSMEM: s_add_u32 m0, s7, 0x300
+; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill
 ; FIXME-TOSMEM-NOT: m0
 
 ; TOSMEM: s_mov_b64 exec,
@@ -80,7 +80,7 @@ endif:
 ; TOSMEM: s_branch
 
 ; TOSMEM: BB{{[0-9]+_[0-9]+}}:
-; TOSMEM: s_add_u32 m0, s7, 0x200
+; TOSMEM: s_add_u32 m0, s7, 0x400
 ; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload
 
 
diff --git a/test/CodeGen/AMDGPU/split-scalar-i64-add.ll b/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
index 5d7d29db3a2f5..59ddd6177d76e 100644
--- a/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
+++ b/test/CodeGen/AMDGPU/split-scalar-i64-add.ll
@@ -7,8 +7,11 @@ declare i32 @llvm.amdgcn.workitem.id.x() readnone
 ; set in vcc, which is undefined since the low scalar half add sets
 ; scc instead.
 
+; FIXME: SIShrinkInstructions should force immediate fold.
+
 ; FUNC-LABEL: {{^}}imp_def_vcc_split_i64_add_0:
-; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, 0x18f, v{{[0-9]+}}
+; SI: s_movk_i32 [[K:s[0-9]+]], 0x18f
+; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, [[K]], v{{[0-9]+}}
 ; SI: v_addc_u32_e32 v{{[0-9]+}}, vcc, 0, v{{[0-9]+}}, vcc
 define amdgpu_kernel void @imp_def_vcc_split_i64_add_0(i64 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %s.val) {
   %v.val = load volatile i32, i32 addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/splitkit.mir b/test/CodeGen/AMDGPU/splitkit.mir
index 41782af40e3cb..45a9c41c38157 100644
--- a/test/CodeGen/AMDGPU/splitkit.mir
+++ b/test/CodeGen/AMDGPU/splitkit.mir
@@ -37,13 +37,13 @@ body: |
 # CHECK: [[REG0:%sgpr[0-9]+]] = COPY %sgpr0
 # CHECK: [[REG1:%sgpr[0-9]+]] = COPY %sgpr2
 # CHECK: S_NOP 0
-# CHECK: S_NOP 0, implicit [[REG0]]
-# CHECK: S_NOP 0, implicit [[REG1]]
-# CHECK: %sgpr0 = COPY [[REG0]]
-# CHECK: %sgpr2 = COPY [[REG1]]
+# CHECK: S_NOP 0, implicit renamable [[REG0]]
+# CHECK: S_NOP 0, implicit renamable [[REG1]]
+# CHECK: %sgpr0 = COPY renamable [[REG0]]
+# CHECK: %sgpr2 = COPY renamable [[REG1]]
 # CHECK: S_NOP
-# CHECK: S_NOP 0, implicit %sgpr0
-# CHECK: S_NOP 0, implicit %sgpr2
+# CHECK: S_NOP 0, implicit renamable %sgpr0
+# CHECK: S_NOP 0, implicit renamable %sgpr2
 name: func1
 tracksRegLiveness: true
 body: |
@@ -67,8 +67,8 @@ body: |
 # Check that copy hoisting out of loops works. This mainly should not crash the
 # compiler when it hoists a subreg copy sequence.
 # CHECK-LABEL: name: splitHoist
-# CHECK: S_NOP 0, implicit-def %sgpr0
-# CHECK: S_NOP 0, implicit-def %sgpr3
+# CHECK: S_NOP 0, implicit-def renamable %sgpr0
+# CHECK: S_NOP 0, implicit-def renamable %sgpr3
 # CHECK-NEXT: SI_SPILL_S128_SAVE
 name: splitHoist
 tracksRegLiveness: true
diff --git a/test/CodeGen/AMDGPU/sra.ll b/test/CodeGen/AMDGPU/sra.ll
index 74618b263bad7..ee8273779e5a9 100644
--- a/test/CodeGen/AMDGPU/sra.ll
+++ b/test/CodeGen/AMDGPU/sra.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() #0
 
diff --git a/test/CodeGen/AMDGPU/srem.ll b/test/CodeGen/AMDGPU/srem.ll
index 51eaf9a960b00..e2b1ea40d2de2 100644
--- a/test/CodeGen/AMDGPU/srem.ll
+++ b/test/CodeGen/AMDGPU/srem.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood < %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood < %s
 
 define amdgpu_kernel void @srem_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %den_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
@@ -22,7 +22,7 @@ define amdgpu_kernel void @srem_i32_4(i32 addrspace(1)* %out, i32 addrspace(1)*
 ; SI: v_mov_b32_e32 [[MAGIC:v[0-9]+]], 0x92492493
 ; SI: v_mul_hi_i32 {{v[0-9]+}}, {{v[0-9]+}}, [[MAGIC]]
 ; SI: v_mul_lo_i32
-; SI: v_sub_i32
+; SI: v_sub_{{[iu]}}32
 ; SI: s_endpgm
 define amdgpu_kernel void @srem_i32_7(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %num = load i32, i32 addrspace(1) * %in
diff --git a/test/CodeGen/AMDGPU/ssubo.ll b/test/CodeGen/AMDGPU/ssubo.ll
index d65c2adc7e202..fee14b48b44ad 100644
--- a/test/CodeGen/AMDGPU/ssubo.ll
+++ b/test/CodeGen/AMDGPU/ssubo.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs< %s
 
 declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
@@ -39,8 +40,8 @@ define amdgpu_kernel void @v_ssubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}s_ssubo_i64:
-; SI: s_sub_u32
-; SI: s_subb_u32
+; GCN: s_sub_u32
+; GCN: s_subb_u32
 define amdgpu_kernel void @s_ssubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
   %ssub = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 %a, i64 %b) nounwind
   %val = extractvalue { i64, i1 } %ssub, 0
@@ -51,8 +52,14 @@ define amdgpu_kernel void @s_ssubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_ssubo_i64:
-; SI: v_sub_i32_e32
-; SI: v_subb_u32_e32
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
 define amdgpu_kernel void @v_ssubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind {
   %a = load i64, i64 addrspace(1)* %aptr, align 4
   %b = load i64, i64 addrspace(1)* %bptr, align 4
diff --git a/test/CodeGen/AMDGPU/stack-size-overflow.ll b/test/CodeGen/AMDGPU/stack-size-overflow.ll
new file mode 100644
index 0000000000000..45a399b058cce
--- /dev/null
+++ b/test/CodeGen/AMDGPU/stack-size-overflow.ll
@@ -0,0 +1,14 @@
+; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+; RUN: not llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+
+declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1) #1
+
+; ERROR: error: stack size limit exceeded (4294967296) in stack_size_limit
+; GCN: ; ScratchSize: 4294967296
+define amdgpu_kernel void @stack_size_limit() #0 {
+entry:
+  %alloca = alloca [1073741823 x i32], align 4
+  %bc = bitcast [1073741823 x i32]* %alloca to i8*
+  call void @llvm.memset.p0i8.i32(i8* %bc, i8 9, i32 1073741823, i32 1, i1 true)
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/stack-slot-color-sgpr-vgpr-spills.mir b/test/CodeGen/AMDGPU/stack-slot-color-sgpr-vgpr-spills.mir
new file mode 100644
index 0000000000000..d5bf6a1eb8c96
--- /dev/null
+++ b/test/CodeGen/AMDGPU/stack-slot-color-sgpr-vgpr-spills.mir
@@ -0,0 +1,34 @@
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -stress-regalloc=1 -start-before=greedy -stop-after=stack-slot-coloring -o - %s | FileCheck  %s
+---
+
+# CHECK-LABEL: name: no_merge_sgpr_vgpr_spill_slot{{$}}
+# CHECK: stack:
+# CHECK:   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4,
+# CHECK-NEXT: stack-id: 0,
+
+# CHECK: - { id: 1, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4,
+# CHECK-NEXT: stack-id: 1,
+
+# CHECK: SI_SPILL_V32_SAVE killed renamable %vgpr0, %stack.0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr5, 0, implicit %exec :: (store 4 into %stack.0)
+# CHECK: %vgpr0 = SI_SPILL_V32_RESTORE %stack.0, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr5, 0, implicit %exec :: (load 4 from %stack.0)
+
+# CHECK: SI_SPILL_S32_SAVE killed renamable %sgpr6, %stack.1, implicit %exec, implicit %sgpr0_sgpr1_sgpr2_sgpr3, implicit %sgpr5, implicit-def dead %m0 :: (store 4 into %stack.1)
+# CHECK: %sgpr6 = SI_SPILL_S32_RESTORE %stack.1, implicit %exec, implicit %sgpr0_sgpr1_sgpr2_sgpr3, implicit %sgpr5, implicit-def dead %m0 :: (load 4 from %stack.1)
+
+name: no_merge_sgpr_vgpr_spill_slot
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: vgpr_32 }
+  - { id: 1, class: sreg_32_xm0_xexec }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: sreg_32_xm0_xexec }
+
+body: |
+  bb.0:
+    %0 = FLAT_LOAD_DWORD undef %vgpr0_vgpr1, 0, 0, 0, implicit %flat_scr, implicit %exec
+    %2 = FLAT_LOAD_DWORD undef %vgpr0_vgpr1, 0, 0, 0, implicit %flat_scr, implicit %exec
+    S_NOP 0, implicit %0
+    %1 = S_LOAD_DWORD_IMM undef %sgpr0_sgpr1, 0, 0
+    %3 = S_LOAD_DWORD_IMM undef %sgpr0_sgpr1, 0, 0
+    S_NOP 0, implicit %1
+...
diff --git a/test/CodeGen/AMDGPU/store-global.ll b/test/CodeGen/AMDGPU/store-global.ll
index f61e524ee2e55..b5af3ead5e7a4 100644
--- a/test/CodeGen/AMDGPU/store-global.ll
+++ b/test/CodeGen/AMDGPU/store-global.ll
@@ -1,7 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
 
 ; FUNC-LABEL: {{^}}store_i1:
 ; EG: MEM_RAT MSKOR
@@ -10,7 +11,8 @@
 ; CM: MEM_RAT MSKOR
 ; CM-NOT: MEM_RAT MSKOR
 
-; GCN: buffer_store_byte
+; SIVI: buffer_store_byte
+; GFX9: global_store_byte
 define amdgpu_kernel void @store_i1(i1 addrspace(1)* %out) {
 entry:
   store i1 true, i1 addrspace(1)* %out
@@ -40,8 +42,8 @@ entry:
 ; EG: MOV T[[RW_GPR]].Y, 0.0
 ; EG: MOV * T[[RW_GPR]].Z, 0.0
 
-; GCN: buffer_store_byte
-
+; SIVI: buffer_store_byte
+; GFX9: global_store_byte
 define amdgpu_kernel void @store_i8(i8 addrspace(1)* %out, i8 %in) {
 entry:
   store i8 %in, i8 addrspace(1)* %out
@@ -74,7 +76,8 @@ entry:
 ; EG: MOV T[[RW_GPR]].Y, 0.0
 ; EG: MOV * T[[RW_GPR]].Z, 0.0
 
-; GCN: buffer_store_short
+; SIVI: buffer_store_short
+; GFX9: global_store_short
 define amdgpu_kernel void @store_i16(i16 addrspace(1)* %out, i16 %in) {
 entry:
   store i16 %in, i16 addrspace(1)* %out
@@ -82,9 +85,12 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_i24:
-; GCN: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
-; GCN-DAG: buffer_store_byte
-; GCN-DAG: buffer_store_short
+; SIVI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16
+; SIVI-DAG: buffer_store_byte
+; SIVI-DAG: buffer_store_short
+
+; GFX9-DAG: global_store_byte_d16_hi v{{\[[0-9]:[0-9]+\]}}, v{{[0-9]+}}, off offset:2
+; GFX9-DAG: global_store_short
 
 ; EG: MEM_RAT MSKOR
 ; EG: MEM_RAT MSKOR
@@ -97,7 +103,8 @@ entry:
 ; FUNC-LABEL: {{^}}store_i25:
 ; GCN: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 0x1ffffff{{$}}
 ; GCN: v_mov_b32_e32 [[VAND:v[0-9]+]], [[AND]]
-; GCN: buffer_store_dword [[VAND]]
+; SIVI: buffer_store_dword [[VAND]]
+; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[VAND]]
 
 ; EG: MEM_RAT_CACHELESS STORE_RAW
 ; EG-NOT: MEM_RAT
@@ -118,7 +125,8 @@ entry:
 ; CM: MEM_RAT MSKOR
 ; CM-NOT: MEM_RAT MSKOR
 
-; GCN: buffer_store_short
+; SIVI: buffer_store_short
+; GFX9: global_store_short
 define amdgpu_kernel void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) {
 entry:
   %0 = trunc <2 x i32> %in to <2 x i8>
@@ -149,7 +157,8 @@ entry:
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dword
+; SIVI: buffer_store_dword
+; GFX9: global_store_dword
 define amdgpu_kernel void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
 entry:
   %0 = trunc <2 x i32> %in to <2 x i16>
@@ -168,8 +177,11 @@ entry:
 ; CM-NOT: MEM_RAT MSKOR
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
-; SI: buffer_store_short
-; SI: buffer_store_short
+; SIVI: buffer_store_short
+; SIVI: buffer_store_short
+
+; GFX9: global_store_short
+; GFX9: global_store_short
 define amdgpu_kernel void @store_v2i16_unaligned(<2 x i16> addrspace(1)* %out, <2 x i32> %in) {
 entry:
   %0 = trunc <2 x i32> %in to <2 x i16>
@@ -182,7 +194,8 @@ entry:
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dword
+; SIVI: buffer_store_dword
+; GFX9: global_store_dword
 define amdgpu_kernel void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   %0 = trunc <4 x i32> %in to <4 x i8>
@@ -244,7 +257,8 @@ entry:
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}}
 
-; GCN: buffer_store_dword
+; SIVI: buffer_store_dword
+; GFX9: global_store_dword
 
 define amdgpu_kernel void @store_f32(float addrspace(1)* %out, float %in) {
   store float %in, float addrspace(1)* %out
@@ -256,7 +270,8 @@ define amdgpu_kernel void @store_f32(float addrspace(1)* %out, float %in) {
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}}
 
-; GCN: buffer_store_dwordx2
+; SIVI: buffer_store_dwordx2
+; GFX9: global_store_dwordx2
 define amdgpu_kernel void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   %0 = trunc <4 x i32> %in to <4 x i16>
@@ -270,7 +285,8 @@ entry:
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dwordx2
+; SIVI: buffer_store_dwordx2
+; GFX9: global_store_dwordx2
 
 define amdgpu_kernel void @store_v2f32(<2 x float> addrspace(1)* %out, float %a, float %b) {
 entry:
@@ -281,8 +297,11 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_v3i32:
-; GCN-DAG: buffer_store_dwordx2
-; GCN-DAG: buffer_store_dword v
+; SIVI-DAG: buffer_store_dwordx2
+; SIVI-DAG: buffer_store_dword v
+
+; GFX9-DAG: global_store_dwordx2
+; GFX9-DAG: global_store_dword v
 
 ; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.[XYZW]}}, {{T[0-9]+\.[XYZW]}},
 ; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XY}}, {{T[0-9]+\.[XYZW]}},
@@ -298,7 +317,8 @@ define amdgpu_kernel void @store_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dwordx4
+; SIVI: buffer_store_dwordx4
+; GFX9: global_store_dwordx4
 define amdgpu_kernel void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(1)* %out
@@ -312,7 +332,8 @@ entry:
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
-; SI: buffer_store_dwordx4
+; SIVI: buffer_store_dwordx4
+; GFX9: global_store_dwordx4
 define amdgpu_kernel void @store_v4i32_unaligned(<4 x i32> addrspace(1)* %out, <4 x i32> %in) {
 entry:
   store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4
@@ -327,7 +348,8 @@ entry:
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dwordx4
+; SIVI: buffer_store_dwordx4
+; GFX9: global_store_dwordx4
 define amdgpu_kernel void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) {
   %1 = load <4 x float>, <4 x float> addrspace(1) * %in
   store <4 x float> %1, <4 x float> addrspace(1)* %out
@@ -339,7 +361,8 @@ define amdgpu_kernel void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x floa
 
 ; CM: MEM_RAT MSKOR
 
-; GCN: buffer_store_byte
+; SIVI: buffer_store_byte
+; GFX9: global_store_byte
 define amdgpu_kernel void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) {
 entry:
   %0 = trunc i64 %in to i8
@@ -349,7 +372,8 @@ entry:
 
 ; FUNC-LABEL: {{^}}store_i64_i16:
 ; EG: MEM_RAT MSKOR
-; GCN: buffer_store_short
+; SIVI: buffer_store_short
+; GFX9: global_store_short
 define amdgpu_kernel void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) {
 entry:
   %0 = trunc i64 %in to i16
@@ -368,7 +392,8 @@ entry:
 ; CM: MEM_RAT_CACHELESS STORE_DWORD
 ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD
 
-; GCN: buffer_store_dwordx2
+; SIVI: buffer_store_dwordx2
+; GFX9: global_store_dwordx2
 define amdgpu_kernel void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 {
 entry:
   %0 = load i32, i32 addrspace(2)* %mem, align 4
@@ -387,7 +412,8 @@ entry:
 
 ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}}, T{{[0-9]+}}.X
 
-; GCN: buffer_store_dwordx4
+; SIVI: buffer_store_dwordx4
+; GFX9: global_store_dwordx4
 define amdgpu_kernel void @i128-const-store(i32 addrspace(1)* %out) {
 entry:
   store i32 1, i32 addrspace(1)* %out, align 4
diff --git a/test/CodeGen/AMDGPU/store-hi16.ll b/test/CodeGen/AMDGPU/store-hi16.ll
new file mode 100644
index 0000000000000..d988ea3549c1f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/store-hi16.ll
@@ -0,0 +1,631 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-sroa=0 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16(i16 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  store i16 %hi, i16 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2f16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2f16(half addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x half>
+  %hi = extractelement <2 x half> %value, i32 1
+  store half %hi, half addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_i32_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_i32_shift(i16 addrspace(1)* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i16
+  store i16 %hi, i16 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16_i8:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_byte v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16_i8(i8 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_i8_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_byte v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_i8_shift(i8 addrspace(1)* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i8
+  store i8 %hi, i8 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16_max_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:4094
+
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+
+; VI: flat_store_short v[0:1], v2{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16_max_offset(i16 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 2047
+  store i16 %hi, i16 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16_min_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_store_short_d16_hi v[0:1], v2, off offset:-4096{{$}}
+
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+
+; VI: flat_store_short v[0:1], v{{[0-9]$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16_min_offset(i16 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16 addrspace(1)* %out, i64 -2048
+  store i16 %hi, i16 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_max_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:4095
+
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; VI: flat_store_byte v[0:1], v{{[0-9]$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16_i8_max_offset(i8 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 4095
+  store i8 %trunc, i8 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_global_hi_v2i16_i8_min_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: global_store_byte_d16_hi v[0:1], v2, off offset:-4095
+
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+
+; VI: flat_store_byte v[0:1], v{{[0-9]$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_global_hi_v2i16_i8_min_offset(i8 addrspace(1)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  %gep = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 -4095
+  store i8 %trunc, i8 addrspace(1)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16(i16 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  store i16 %hi, i16 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2f16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2f16(half addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x half>
+  %hi = extractelement <2 x half> %value, i32 1
+  store half %hi, half addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_i32_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_short v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_i32_shift(i16 addrspace(4)* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i16
+  store i16 %hi, i16 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_byte v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16_i8(i8 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_i8_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: flat_store_byte v[0:1], v2
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_i8_shift(i8 addrspace(4)* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i8
+  store i8 %hi, i8 addrspace(4)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16_max_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2 offset:4094{{$}}
+
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; VI: flat_store_short v[0:1], v2{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16_max_offset(i16 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16 addrspace(4)* %out, i64 2047
+  store i16 %hi, i16 addrspace(4)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16_neg_offset:
+; GCN: s_waitcnt
+; GCN: v_add{{(_co)?}}_{{i|u}}32_e32
+; VI: v_addc_u32_e32
+; GFX9: v_addc_co_u32_e32
+
+; GFX9-NEXT: flat_store_short_d16_hi v[0:1], v2{{$}}
+; VI: flat_store_short v[0:1], v2{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16_neg_offset(i16 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16 addrspace(4)* %out, i64 -1023
+  store i16 %hi, i16 addrspace(4)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_max_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2 offset:4095{{$}}
+
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; VI-DAG: v_add_u32_e32
+; VI-DAG: v_addc_u32_e32
+; VI: flat_store_byte v[0:1], v2{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16_i8_max_offset(i8 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  %gep = getelementptr inbounds i8, i8 addrspace(4)* %out, i64 4095
+  store i8 %trunc, i8 addrspace(4)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_flat_hi_v2i16_i8_neg_offset:
+; GCN: s_waitcnt
+; GCN-DAG: v_add{{(_co)?}}_{{i|u}}32_e32
+; VI-DAG: v_addc_u32_e32
+; GFX9-DAG: v_addc_co_u32_e32
+
+; GFX9-NEXT: flat_store_byte_d16_hi v[0:1], v2{{$}}
+; VI-DAG: v_lshrrev_b32_e32 v2, 16, v2
+; VI: flat_store_byte v[0:1], v2{{$}}
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_flat_hi_v2i16_i8_neg_offset(i8 addrspace(4)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  %gep = getelementptr inbounds i8, i8 addrspace(4)* %out, i64 -4095
+  store i8 %trunc, i8 addrspace(4)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2i16(i16* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  store i16 %hi, i16* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2f16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2f16(half* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x half>
+  %hi = extractelement <2 x half> %value, i32 1
+  store half %hi, half* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_i32_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_short_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT: buffer_store_short v1, v0, s[0:3], s4 offen{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_i32_shift(i16* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i16
+  store i16 %hi, i16* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_i8:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2i16_i8(i8* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_i8_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_byte_d16_hi v1, v0, s[0:3], s4 offen{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; VI-NEXT: buffer_store_byte v1, v0, s[0:3], s4 offen{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_i8_shift(i8* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i8
+  store i8 %hi, i8* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_max_offset:
+; GCN: s_waitcnt
+; GFX9: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094{{$}}
+
+; VI: v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT: buffer_store_short v0, off, s[0:3], s5 offset:4094{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2i16_max_offset(i16* byval %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16* %out, i64 2045
+  store i16 %hi, i16* %gep
+  ret void
+}
+
+
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_nooff:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s4{{$}}
+
+; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; VI-NEXT: buffer_store_short v0, off, s[0:3], s4{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2i16_nooff(i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  store volatile i16 %hi, i16* null
+  ret void
+}
+
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_nooff:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s4{{$}}
+
+; VI: v_lshrrev_b32_e32 v0, 16, v0
+; VI: buffer_store_byte v0, off, s[0:3], s4{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_private_hi_v2i16_i8_nooff(i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  store volatile i8 %trunc, i8* null
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_local_hi_v2i16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: ds_write_b16 v0, v1
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_local_hi_v2i16(i16 addrspace(3)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  store i16 %hi, i16 addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_local_hi_v2f16:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: ds_write_b16 v0, v1
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_local_hi_v2f16(half addrspace(3)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x half>
+  %hi = extractelement <2 x half> %value, i32 1
+  store half %hi, half addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_local_hi_i32_shift:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: ds_write_b16_d16_hi v0, v1{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: ds_write_b16 v0, v1
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_local_hi_i32_shift(i16 addrspace(3)* %out, i32 %value) #0 {
+entry:
+  %hi32 = lshr i32 %value, 16
+  %hi = trunc i32 %hi32 to i16
+  store i16 %hi, i16 addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_local_hi_v2i16_i8:
+; GCN: s_waitcnt
+
+; GFX9-NEXT: ds_write_b8_d16_hi v0, v1{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: ds_write_b8 v0, v1
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_local_hi_v2i16_i8(i8 addrspace(3)* %out, i32 %arg) #0 {
+entry:
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8 addrspace(3)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_local_hi_v2i16_max_offset:
+; GCN: s_waitcnt
+; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:65534{{$}}
+
+; VI: v_lshrrev_b32_e32 v1, 16, v1
+; VI: ds_write_b16 v0, v1 offset:65534{{$}}
+
+; GCN-NEXT: s_waitcnt
+; GCN-NEXT: s_setpc_b64
+define void @store_local_hi_v2i16_max_offset(i16 addrspace(3)* %out, i32 %arg) #0 {
+entry:
+  ; FIXME: ABI for pre-gfx9
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds i16, i16 addrspace(3)* %out, i64 32767
+  store i16 %hi, i16 addrspace(3)* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_to_offset:
+; GCN: s_waitcnt
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_store_short_d16_hi v0, off, s[0:3], s5 offset:4094
+define void @store_private_hi_v2i16_to_offset(i32 %arg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i16], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds [4096 x i16], [4096 x i16]* %obj1, i32 0, i32 2025
+  store i16 %hi, i16* %gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_hi_v2i16_i8_to_offset:
+; GCN: s_waitcnt
+; GFX9: buffer_store_dword
+; GFX9-NEXT: buffer_store_byte_d16_hi v0, off, s[0:3], s5 offset:4095
+define void @store_private_hi_v2i16_i8_to_offset(i32 %arg) #0 {
+entry:
+  %obj0 = alloca [10 x i32], align 4
+  %obj1 = alloca [4096 x i8], align 2
+  %bc = bitcast [10 x i32]* %obj0 to i32*
+  store volatile i32 123, i32* %bc
+  %value = bitcast i32 %arg to <2 x i16>
+  %hi = extractelement <2 x i16> %value, i32 1
+  %gep = getelementptr inbounds [4096 x i8], [4096 x i8]* %obj1, i32 0, i32 4051
+  %trunc = trunc i16 %hi to i8
+  store i8 %trunc, i8* %gep
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/store-local.ll b/test/CodeGen/AMDGPU/store-local.ll
index c144bf2aa878c..96d5e06a9e96d 100644
--- a/test/CodeGen/AMDGPU/store-local.ll
+++ b/test/CodeGen/AMDGPU/store-local.ll
@@ -1,9 +1,13 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=cayman < %s | FileCheck -check-prefixes=CM,FUNC %s
 
 ; FUNC-LABEL: {{^}}store_local_i1:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_BYTE_WRITE
 
 ; CM: LDS_BYTE_WRITE
@@ -16,6 +20,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_i8:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_BYTE_WRITE
 
 ; CM: LDS_BYTE_WRITE
@@ -27,6 +34,9 @@ define amdgpu_kernel void @store_local_i8(i8 addrspace(3)* %out, i8 %in) {
 }
 
 ; FUNC-LABEL: {{^}}store_local_i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_SHORT_WRITE
 
 ; CM: LDS_SHORT_WRITE
@@ -38,6 +48,9 @@ define amdgpu_kernel void @store_local_i16(i16 addrspace(3)* %out, i16 %in) {
 }
 
 ; FUNC-LABEL: {{^}}store_local_v2i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRITE
 
 ; CM: LDS_WRITE
@@ -50,6 +63,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v4i8:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRITE
 
 ; CM: LDS_WRITE
@@ -62,6 +78,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v4i8_unaligned:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_BYTE_WRITE
 ; EG: LDS_BYTE_WRITE
 ; EG: LDS_BYTE_WRITE
@@ -85,6 +104,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v4i8_halfaligned:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_SHORT_WRITE
 ; EG: LDS_SHORT_WRITE
 ; EG-NOT: LDS_WRITE
@@ -102,6 +124,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v2i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
 ; EG-NOT: LDS_WRITE
@@ -118,6 +143,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v4i32:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
@@ -136,6 +164,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_v4i32_align4:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
 ; EG: LDS_WRITE
@@ -155,6 +186,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_i64_i8:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_BYTE_WRITE
 ; GCN: ds_write_b8
 define amdgpu_kernel void @store_local_i64_i8(i8 addrspace(3)* %out, i64 %in) {
@@ -165,6 +199,9 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_local_i64_i16:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
 ; EG: LDS_SHORT_WRITE
 ; GCN: ds_write_b16
 define amdgpu_kernel void @store_local_i64_i16(i16 addrspace(3)* %out, i64 %in) {
diff --git a/test/CodeGen/AMDGPU/store-vector-ptrs.ll b/test/CodeGen/AMDGPU/store-vector-ptrs.ll
index 507f07dee0524..26f8bde6d8ded 100644
--- a/test/CodeGen/AMDGPU/store-vector-ptrs.ll
+++ b/test/CodeGen/AMDGPU/store-vector-ptrs.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s
 
 ; This tests for a bug that caused a crash in
diff --git a/test/CodeGen/AMDGPU/store-weird-sizes.ll b/test/CodeGen/AMDGPU/store-weird-sizes.ll
new file mode 100644
index 0000000000000..fd82dccb0fac6
--- /dev/null
+++ b/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -0,0 +1,56 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CIVI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+
+; GCN-LABEL: {{^}}local_store_i56:
+; GCN-DAG: ds_write_b8 v0, v{{[0-9]+}} offset:6
+; GCN-DAG: ds_write_b16 v0, v{{[0-9]+}} offset:4
+; GCN-DAG: ds_write_b32 v0, v{{[0-9]+$}}
+define void @local_store_i56(i56 addrspace(3)* %ptr, i56 %arg) #0 {
+  store i56 %arg, i56 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}local_store_i55:
+; GCN-DAG: ds_write_b8 v0, v{{[0-9]+}} offset:6
+; GCN-DAG: ds_write_b16 v0, v{{[0-9]+}} offset:4
+; GCN-DAG: ds_write_b32 v0, v{{[0-9]+$}}
+define amdgpu_kernel void @local_store_i55(i55 addrspace(3)* %ptr, i55 %arg) #0 {
+  store i55 %arg, i55 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}local_store_i48:
+; GCN-DAG: ds_write_b16 v0, v{{[0-9]+}} offset:4
+; GCN-DAG: ds_write_b32 v0, v{{[0-9]+$}}
+define amdgpu_kernel void @local_store_i48(i48 addrspace(3)* %ptr, i48 %arg) #0 {
+  store i48 %arg, i48 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}local_store_i65:
+; GCN-DAG: ds_write_b8 v{{[0-9]+}}, v{{[0-9]+}} offset:8
+; GCN-DAG: ds_write_b64
+define amdgpu_kernel void @local_store_i65(i65 addrspace(3)* %ptr, i65 %arg) #0 {
+  store i65 %arg, i65 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}local_store_i13:
+; GCN: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0x1fff, v1
+; GCN: ds_write_b16 v0, [[TRUNC]]
+define void @local_store_i13(i13 addrspace(3)* %ptr, i13 %arg) #0 {
+  store i13 %arg, i13 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+; GCN-LABEL: {{^}}local_store_i17:
+; GCN: ds_write_b16 v0
+; CIVI: ds_write_b8 v0, v{{[0-9]+}} offset:2
+; GFX9: ds_write_b8_d16_hi v0, v{{[0-9]+}} offset:2
+define void @local_store_i17(i17 addrspace(3)* %ptr, i17 %arg) #0 {
+  store i17 %arg, i17 addrspace(3)* %ptr, align 8
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/stress-calls.ll b/test/CodeGen/AMDGPU/stress-calls.ll
new file mode 100644
index 0000000000000..480d40d67d483
--- /dev/null
+++ b/test/CodeGen/AMDGPU/stress-calls.ll
@@ -0,0 +1,36 @@
+; RUN: opt -S -amdgpu-stress-function-calls -amdgpu-always-inline %s | FileCheck %s
+
+; CHECK: define internal fastcc i32 @alwaysinline_func(i32 %a) #0 {
+define internal fastcc i32 @alwaysinline_func(i32 %a) alwaysinline {
+entry:
+  %tmp0 = add i32 %a, 1
+  ret i32 %tmp0
+}
+
+; CHECK: define internal fastcc i32 @noinline_func(i32 %a) #1 {
+define internal fastcc i32 @noinline_func(i32 %a) noinline {
+entry:
+  %tmp0 = add i32 %a, 2
+  ret i32 %tmp0
+}
+
+; CHECK: define internal fastcc i32 @unmarked_func(i32 %a) #1 {
+define internal fastcc i32 @unmarked_func(i32 %a) {
+entry:
+  %tmp0 = add i32 %a, 3
+  ret i32 %tmp0
+}
+
+define amdgpu_kernel void @kernel(i32 addrspace(1)* %out) {
+entry:
+  %tmp0 = call i32 @alwaysinline_func(i32 1)
+  store volatile i32 %tmp0, i32 addrspace(1)* %out
+  %tmp1 = call i32 @noinline_func(i32 1)
+  store volatile i32 %tmp1, i32 addrspace(1)* %out
+  %tmp2 = call i32 @unmarked_func(i32 1)
+  store volatile i32 %tmp2, i32 addrspace(1)* %out
+  ret void
+}
+
+; CHECK: attributes #0 = { alwaysinline }
+; CHECK: attributes #1 = { noinline }
diff --git a/test/CodeGen/AMDGPU/sub.i16.ll b/test/CodeGen/AMDGPU/sub.i16.ll
index 14bedceed6eee..3ad091a91d68a 100644
--- a/test/CodeGen/AMDGPU/sub.i16.ll
+++ b/test/CodeGen/AMDGPU/sub.i16.ll
@@ -85,9 +85,9 @@ define amdgpu_kernel void @v_test_sub_i16_zext_to_i32(i32 addrspace(1)* %out, i1
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_i16_zext_to_i64:
-; VI: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
+; VI: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0
 ; VI-DAG: v_sub_u16_e32 v[[ADD:[0-9]+]], [[A]], [[B]]
 ; VI: buffer_store_dwordx2 v{{\[}}[[ADD]]:[[VZERO]]{{\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0{{$}}
 define amdgpu_kernel void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@@ -107,7 +107,7 @@ define amdgpu_kernel void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i32:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
+; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
 ; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: buffer_store_dword [[SEXT]]
 define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 {
@@ -127,7 +127,7 @@ define amdgpu_kernel void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i1
 ; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i64:
 ; VI: flat_load_ushort [[A:v[0-9]+]]
 ; VI: flat_load_ushort [[B:v[0-9]+]]
-; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[A]], [[B]]
+; VI: v_sub_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]]
 ; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16
 ; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]]
 ; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
diff --git a/test/CodeGen/AMDGPU/sub.ll b/test/CodeGen/AMDGPU/sub.ll
index 46f1b120f2127..908d13eb01789 100644
--- a/test/CodeGen/AMDGPU/sub.ll
+++ b/test/CodeGen/AMDGPU/sub.ll
@@ -1,13 +1,34 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
-
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=EG,FUNC %s
 
 declare i32 @llvm.r600.read.tidig.x() readnone
 
+; FUNC-LABEL: {{^}}s_sub_i32:
+; GCN: s_load_dword [[A:s[0-9]+]]
+; GCN: s_load_dword [[B:s[0-9]+]]
+; GCN: s_sub_i32 s{{[0-9]+}}, [[A]], [[B]]
+define amdgpu_kernel void @s_sub_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) {
+  %result = sub i32 %a, %b
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}s_sub_imm_i32:
+; GCN: s_load_dword [[A:s[0-9]+]]
+; GCN: s_sub_i32 s{{[0-9]+}}, 0x4d2, [[A]]
+define amdgpu_kernel void @s_sub_imm_i32(i32 addrspace(1)* %out, i32 %a) {
+  %result = sub i32 1234, %a
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
 ; FUNC-LABEL: {{^}}test_sub_i32:
 ; EG: SUB_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
 
 ; SI: v_subrev_i32_e32 v{{[0-9]+, vcc, v[0-9]+, v[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 define amdgpu_kernel void @test_sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
   %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
   %a = load i32, i32 addrspace(1)* %in
@@ -17,6 +38,17 @@ define amdgpu_kernel void @test_sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)
   ret void
 }
 
+; FUNC-LABEL: {{^}}test_sub_imm_i32:
+; EG: SUB_INT
+
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc, 0x7b, v{{[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+}}, 0x7b, v{{[0-9]+}}
+define amdgpu_kernel void @test_sub_imm_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %a = load i32, i32 addrspace(1)* %in
+  %result = sub i32 123, %a
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
 
 ; FUNC-LABEL: {{^}}test_sub_v2i32:
 ; EG: SUB_INT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}
@@ -25,6 +57,8 @@ define amdgpu_kernel void @test_sub_i32(i32 addrspace(1)* %out, i32 addrspace(1)
 ; SI: v_sub_i32_e32 v{{[0-9]+, vcc, v[0-9]+, v[0-9]+}}
 ; SI: v_sub_i32_e32 v{{[0-9]+, vcc, v[0-9]+, v[0-9]+}}
 
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 define amdgpu_kernel void @test_sub_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1
   %a = load <2 x i32>, <2 x i32> addrspace(1) * %in
@@ -45,6 +79,10 @@ define amdgpu_kernel void @test_sub_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32
 ; SI: v_sub_i32_e32 v{{[0-9]+, vcc, v[0-9]+, v[0-9]+}}
 ; SI: v_sub_i32_e32 v{{[0-9]+, vcc, v[0-9]+, v[0-9]+}}
 
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; GFX9: v_sub_u32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 define amdgpu_kernel void @test_sub_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) {
   %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1
   %a = load <4 x i32>, <4 x i32> addrspace(1) * %in
@@ -54,49 +92,58 @@ define amdgpu_kernel void @test_sub_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32
   ret void
 }
 
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; FUNC-LABEL: {{^}}test_sub_i16:
+; SI: v_subrev_i32_e32 v{{[0-9]+}}, vcc,
+; GFX89: v_sub_u16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
 define amdgpu_kernel void @test_sub_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
-    %b_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1
-    %a = load i16, i16 addrspace(1)* %in 
-    %b = load i16, i16 addrspace(1)* %b_ptr
-    %result = sub i16 %a, %b
-    store i16 %result, i16 addrspace(1)* %out
-    ret void
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep = getelementptr i16, i16 addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr i16, i16 addrspace(1)* %gep, i32 1
+  %a = load volatile i16, i16 addrspace(1)* %gep
+  %b = load volatile i16, i16 addrspace(1)* %b_ptr
+  %result = sub i16 %a, %b
+  store i16 %result, i16 addrspace(1)* %out
+  ret void
 }
 
 ; FUNC-LABEL: {{^}}test_sub_v2i16:
+; VI: v_sub_u16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; VI: v_sub_u16_sdwa v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
+; GFX9: v_pk_sub_i16
 define amdgpu_kernel void @test_sub_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) {
-    %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i16 1
-    %a = load <2 x i16>, <2 x i16> addrspace(1) * %in 
-    %b = load <2 x i16>, <2 x i16> addrspace(1) * %b_ptr
-    %result = sub <2 x i16> %a, %b
-    store <2 x i16> %result, <2 x i16> addrspace(1)* %out
-    ret void
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %gep, i16 1
+  %a = load <2 x i16>, <2 x i16> addrspace(1)* %gep
+  %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr
+  %result = sub <2 x i16> %a, %b
+  store <2 x i16> %result, <2 x i16> addrspace(1)* %out
+  ret void
 }
 
 ; FUNC-LABEL: {{^}}test_sub_v4i16:
+; VI: v_sub_u16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; VI: v_sub_u16_sdwa v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; VI: v_sub_u16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
+; VI: v_sub_u16_sdwa v{{[0-9]+, v[0-9]+, v[0-9]+}}
 
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}}
-
+; GFX9: v_pk_sub_i16
+; GFX9: v_pk_sub_i16
 define amdgpu_kernel void @test_sub_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) {
-    %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i16 1
-    %a = load <4 x i16>, <4 x i16> addrspace(1) * %in 
-    %b = load <4 x i16>, <4 x i16> addrspace(1) * %b_ptr
-    %result = sub <4 x i16> %a, %b
-    store <4 x i16> %result, <4 x i16> addrspace(1)* %out
-    ret void
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i32 %tid
+  %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %gep, i16 1
+  %a = load <4 x i16>, <4 x i16> addrspace(1) * %gep
+  %b = load <4 x i16>, <4 x i16> addrspace(1) * %b_ptr
+  %result = sub <4 x i16> %a, %b
+  store <4 x i16> %result, <4 x i16> addrspace(1)* %out
+  ret void
 }
 
 ; FUNC-LABEL: {{^}}s_sub_i64:
-; SI: s_sub_u32
-; SI: s_subb_u32
+; GCN: s_sub_u32
+; GCN: s_subb_u32
 
 ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY
 ; EG-DAG: SUB_INT {{[* ]*}}
@@ -113,6 +160,12 @@ define amdgpu_kernel void @s_sub_i64(i64 addrspace(1)* noalias %out, i64 %a, i64
 ; SI: v_sub_i32_e32
 ; SI: v_subb_u32_e32
 
+; VI: v_sub_u32_e32
+; VI: v_subb_u32_e32
+
+; GFX9: v_sub_co_u32_e32
+; GFX9: v_subb_co_u32_e32
+
 ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY
 ; EG-DAG: SUB_INT {{[* ]*}}
 ; EG-DAG: SUBB_UINT
@@ -130,10 +183,20 @@ define amdgpu_kernel void @v_sub_i64(i64 addrspace(1)* noalias %out, i64 addrspa
 }
 
 ; FUNC-LABEL: {{^}}v_test_sub_v2i64:
-; SI: v_sub_i32_e32
-; SI: v_subb_u32_e32
-; SI: v_sub_i32_e32
-; SI: v_subb_u32_e32
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
 define amdgpu_kernel void @v_test_sub_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* noalias %inA, <2 x i64> addrspace(1)* noalias %inB) {
   %tid = call i32 @llvm.r600.read.tidig.x() readnone
   %a_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %inA, i32 %tid
@@ -146,14 +209,32 @@ define amdgpu_kernel void @v_test_sub_v2i64(<2 x i64> addrspace(1)* %out, <2 x i
 }
 
 ; FUNC-LABEL: {{^}}v_test_sub_v4i64:
-; SI: v_subrev_i32_e32
-; SI: v_subb_u32_e32
-; SI: v_subrev_i32_e32
-; SI: v_subb_u32_e32
-; SI: v_subrev_i32_e32
-; SI: v_subb_u32_e32
-; SI: v_subrev_i32_e32
-; SI: v_subb_u32_e32
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc,
+; SI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc,
+; VI: v_subb_u32_e32 v{{[0-9]+}}, vcc,
+
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc,
+; GFX9: v_subb_co_u32_e32 v{{[0-9]+}}, vcc,
 define amdgpu_kernel void @v_test_sub_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* noalias %inA, <4 x i64> addrspace(1)* noalias %inB) {
   %tid = call i32 @llvm.r600.read.tidig.x() readnone
   %a_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %inA, i32 %tid
diff --git a/test/CodeGen/AMDGPU/sub.v2i16.ll b/test/CodeGen/AMDGPU/sub.v2i16.ll
index 8d5c8b64efb83..998cfdf395c4d 100644
--- a/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -enable-packed-inlinable-literals < %s | FileCheck -check-prefix=GFX9 -check-prefix=GCN %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
@@ -49,8 +49,8 @@ define amdgpu_kernel void @s_test_sub_self_v2i16(<2 x i16> addrspace(1)* %out, <
 ; GCN-LABEL: {{^}}s_test_sub_v2i16_kernarg:
 ; GFX9: v_pk_sub_i16 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}
 
-; VI: v_subrev_i32_e32
-; VI: v_subrev_i32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI: v_subrev_u32_e32
+; VI: v_subrev_u32_sdwa v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 define amdgpu_kernel void @s_test_sub_v2i16_kernarg(<2 x i16> addrspace(1)* %out, <2 x i16> %a, <2 x i16> %b) #1 {
   %add = sub <2 x i16> %a, %b
   store <2 x i16> %add, <2 x i16> addrspace(1)* %out
@@ -152,18 +152,19 @@ define amdgpu_kernel void @v_test_sub_v2i16_inline_fp_split(<2 x i16> addrspace(
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_v2i16_zext_to_v2i32:
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_sub_i16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_and_b32_e32 v[[ELT0:[0-9]+]], 0xffff, [[ADD]]
 ; GFX9-DAG: v_lshrrev_b32_e32 v[[ELT1:[0-9]+]], 16, [[ADD]]
 ; GFX9: buffer_store_dwordx2 v{{\[}}[[ELT0]]:[[ELT1]]{{\]}}
 
-; VI: flat_load_ushort v[[A_HI:[0-9]+]]
 ; VI: flat_load_ushort v[[A_LO:[0-9]+]]
-; VI: flat_load_ushort v[[B_HI:[0-9]+]]
+; VI: flat_load_ushort v[[A_HI:[0-9]+]]
+
 ; VI: flat_load_ushort v[[B_LO:[0-9]+]]
+; VI: flat_load_ushort v[[B_HI:[0-9]+]]
 
 ; VI: v_sub_u16_e32 v[[ADD_HI:[0-9]+]], v[[A_HI]], v[[B_HI]]
 ; VI-NOT: and
@@ -188,8 +189,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i32(<2 x i32> addrspace(1)
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_v2i16_zext_to_v2i64:
 ; GFX9: v_mov_b32_e32 v{{[0-9]+}}, 0{{$}}
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_sub_i16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_and_b32_e32 v[[ELT0:[0-9]+]], 0xffff, [[ADD]]
@@ -220,8 +221,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(<2 x i64> addrspace(1)
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_v2i16_sext_to_v2i32:
-; GFX9: flat_load_dword [[A:v[0-9]+]]
-; GFX9: flat_load_dword [[B:v[0-9]+]]
+; GFX9: global_load_dword [[A:v[0-9]+]]
+; GFX9: global_load_dword [[B:v[0-9]+]]
 
 ; GFX9: v_pk_sub_i16 [[ADD:v[0-9]+]], [[A]], [[B]]
 ; GFX9-DAG: v_bfe_i32 v[[ELT0:[0-9]+]], [[ADD]], 0, 16
@@ -246,8 +247,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_sext_to_v2i32(<2 x i32> addrspace(1)
 
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; GCN-LABEL: {{^}}v_test_sub_v2i16_sext_to_v2i64:
-; GCN: flat_load_dword
-; GCN: flat_load_dword
+; GCN: {{flat|global}}_load_dword
+; GCN: {{flat|global}}_load_dword
 
 ; GFX9: v_pk_sub_i16
 ; GFX9: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
index 35615c40d498d..6a9d8ea009380 100644
--- a/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
+++ b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=SI -o - %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -o - %s | FileCheck %s
 ; Don't crash when the use of an undefined value is only detected by the
 ; register coalescer because it is hidden with subregister insert/extract.
 target triple="amdgcn--"
@@ -10,7 +10,6 @@ target triple="amdgcn--"
 ; CHECK: v_mbcnt_lo_u32_b32_e64
 ; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
 ; CHECK-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; CHECK-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
 ; BB0_1:
 ; CHECK: s_load_dword s0, s[0:1], 0xa
 ; CHECK-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/test/CodeGen/AMDGPU/subreg-intervals.mir b/test/CodeGen/AMDGPU/subreg-intervals.mir
index 62816da25b2c4..2d353b8138e3f 100644
--- a/test/CodeGen/AMDGPU/subreg-intervals.mir
+++ b/test/CodeGen/AMDGPU/subreg-intervals.mir
@@ -2,11 +2,11 @@
 # REQUIRES: asserts
 
 # CHECK: INTERVALS
-# CHECK: vreg0
+# CHECK: %0
 # CHECK-LABEL: Machine code for function test0:
 
 # CHECK: INTERVALS
-# CHECK: vreg0
+# CHECK: %0
 # CHECK-LABEL: Machine code for function test1:
 
 --- |
diff --git a/test/CodeGen/AMDGPU/subreg_interference.mir b/test/CodeGen/AMDGPU/subreg_interference.mir
index 6fc22c8d189f0..3575e41c2b78e 100644
--- a/test/CodeGen/AMDGPU/subreg_interference.mir
+++ b/test/CodeGen/AMDGPU/subreg_interference.mir
@@ -12,12 +12,12 @@
 # sgpr0-sgpr3.
 #
 # CHECK-LABEL: func0
-# CHECK: S_NOP 0, implicit-def %sgpr0
-# CHECK: S_NOP 0, implicit-def %sgpr3
-# CHECK: S_NOP 0, implicit-def %sgpr1
-# CHECK: S_NOP 0, implicit-def %sgpr2
-# CHECK: S_NOP 0, implicit %sgpr0, implicit %sgpr3
-# CHECK: S_NOP 0, implicit %sgpr1, implicit %sgpr2
+# CHECK: S_NOP 0, implicit-def renamable %sgpr0
+# CHECK: S_NOP 0, implicit-def renamable %sgpr3
+# CHECK: S_NOP 0, implicit-def renamable %sgpr1
+# CHECK: S_NOP 0, implicit-def renamable %sgpr2
+# CHECK: S_NOP 0, implicit renamable %sgpr0, implicit renamable %sgpr3
+# CHECK: S_NOP 0, implicit renamable %sgpr1, implicit renamable %sgpr2
 name: func0
 body: |
   bb.0:
diff --git a/test/CodeGen/AMDGPU/syncscopes.ll b/test/CodeGen/AMDGPU/syncscopes.ll
index 3741ce788993e..5cea1588d4bbd 100644
--- a/test/CodeGen/AMDGPU/syncscopes.ll
+++ b/test/CodeGen/AMDGPU/syncscopes.ll
@@ -1,9 +1,9 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -stop-before=si-debugger-insert-nops < %s | FileCheck --check-prefix=GCN %s
 
 ; GCN-LABEL: name: syncscopes
-; GCN: FLAT_STORE_DWORD killed %vgpr1_vgpr2, killed %vgpr0, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("agent") seq_cst 4 into %ir.agent_out)
-; GCN: FLAT_STORE_DWORD killed %vgpr4_vgpr5, killed %vgpr3, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("workgroup") seq_cst 4 into %ir.workgroup_out)
-; GCN: FLAT_STORE_DWORD killed %vgpr7_vgpr8, killed %vgpr6, 0, -1, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("wavefront") seq_cst 4 into %ir.wavefront_out)
+; GCN: FLAT_STORE_DWORD killed renamable %vgpr1_vgpr2, killed renamable %vgpr0, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("agent") seq_cst 4 into %ir.agent_out)
+; GCN: FLAT_STORE_DWORD killed renamable %vgpr4_vgpr5, killed renamable %vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("workgroup") seq_cst 4 into %ir.workgroup_out)
+; GCN: FLAT_STORE_DWORD killed renamable %vgpr7_vgpr8, killed renamable %vgpr6, 0, 0, 0, implicit %exec, implicit %flat_scr :: (volatile store syncscope("wavefront") seq_cst 4 into %ir.wavefront_out)
 define void @syncscopes(
     i32 %agent,
     i32 addrspace(4)* %agent_out,
diff --git a/test/CodeGen/AMDGPU/tail-call-cgp.ll b/test/CodeGen/AMDGPU/tail-call-cgp.ll
new file mode 100644
index 0000000000000..6131ab2ae432b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/tail-call-cgp.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -codegenprepare %s | FileCheck %s
+
+define internal fastcc void @callee(i32* nocapture %p, i32 %a) #0 {
+  store volatile i32 %a, i32* %p, align 4
+  ret void
+}
+
+; CHECK-LABEL: @func_caller(
+; CHECK: tail call fastcc void @callee(
+; CHECK-NEXT: ret void
+; CHECK: ret void
+define void @func_caller(i32* nocapture %p, i32 %a, i32 %b) #0 {
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %bb, label %ret
+
+bb:
+  tail call fastcc void @callee(i32* %p, i32 %a)
+  br label %ret
+
+ret:
+  ret void
+}
+
+; CHECK-LABEL: @kernel_caller(
+; CHECK: tail call fastcc void @callee(
+; CHECK-NEXT: br label %ret
+
+; CHECK: ret void
+define amdgpu_kernel void @kernel_caller(i32* nocapture %p, i32 %a, i32 %b) #0 {
+entry:
+  %cmp = icmp eq i32 %b, 0
+  br i1 %cmp, label %bb, label %ret
+
+bb:
+  tail call fastcc void @callee(i32* %p, i32 %a)
+  br label %ret
+
+ret:
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/trunc-cmp-constant.ll b/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
index cb8d365503310..e1644e4a6d241 100644
--- a/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
+++ b/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
@@ -94,7 +94,7 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_0(i1 addrspace(1)* %o
 
 ; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_ne_0:
 ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]]
-; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]]
+; SI: v_and_b32_e32 [[RESULT:v[0-9]+]], 1, [[LOAD]]
 ; SI: buffer_store_byte [[RESULT]]
 define amdgpu_kernel void @zextload_i1_to_i32_trunc_cmp_ne_0(i1 addrspace(1)* %out, i1 addrspace(1)* %in) nounwind {
   %load = load i1, i1 addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll b/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
index 3dbc10d2e9b56..d280be5eba46e 100644
--- a/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
+++ b/test/CodeGen/AMDGPU/trunc-vector-store-assertion-failure.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s
+; RUN: llc < %s -march=r600 -mtriple=r600---amdgiz -mcpu=redwood | FileCheck %s
 
 ; This tests for a bug in the SelectionDAG where custom lowered truncated
 ; vector stores at the end of a basic block were not being added to the
diff --git a/test/CodeGen/AMDGPU/trunc.ll b/test/CodeGen/AMDGPU/trunc.ll
index da038f4b05972..b390958ab4cf3 100644
--- a/test/CodeGen/AMDGPU/trunc.ll
+++ b/test/CodeGen/AMDGPU/trunc.ll
@@ -1,6 +1,6 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -verify-machineinstrs< %s | FileCheck -check-prefix=GCN -check-prefix=VI  %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -verify-machineinstrs< %s | FileCheck -check-prefix=GCN -check-prefix=VI  %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
 
 declare i32 @llvm.r600.read.tidig.x() nounwind readnone
 
diff --git a/test/CodeGen/AMDGPU/tti-unroll-prefs.ll b/test/CodeGen/AMDGPU/tti-unroll-prefs.ll
index 7c369a3127617..6f9f9fc6837f1 100644
--- a/test/CodeGen/AMDGPU/tti-unroll-prefs.ll
+++ b/test/CodeGen/AMDGPU/tti-unroll-prefs.ll
@@ -1,4 +1,4 @@
-; RUN: opt -loop-unroll -S -mtriple=amdgcn-- -mcpu=SI %s | FileCheck %s
+; RUN: opt -loop-unroll -S -mtriple=amdgcn-- -mcpu=tahiti %s | FileCheck %s
 
 ; This IR comes from this OpenCL C code:
 ;
diff --git a/test/CodeGen/AMDGPU/twoaddr-mad.mir b/test/CodeGen/AMDGPU/twoaddr-mad.mir
new file mode 100644
index 0000000000000..707676d848976
--- /dev/null
+++ b/test/CodeGen/AMDGPU/twoaddr-mad.mir
@@ -0,0 +1,190 @@
+# RUN: llc -march=amdgcn %s -run-pass twoaddressinstruction -verify-machineinstrs -o - | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: test_madmk_reg_imm_f32
+# GCN: V_MADMK_F32 killed %0.sub0, 1078523331, killed %1, implicit %exec
+---
+name:            test_madmk_reg_imm_f32
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = COPY %0.sub1
+    %2 = V_MOV_B32_e32 1078523331, implicit %exec
+    %3 = V_MAC_F32_e32 killed %0.sub0, %2, killed %1, implicit %exec
+
+...
+
+# GCN-LABEL: name: test_madmk_imm_reg_f32
+# GCN: V_MADMK_F32 killed %0.sub0, 1078523331, killed %1, implicit %exec
+---
+name:            test_madmk_imm_reg_f32
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = COPY %0.sub1
+    %2 = V_MOV_B32_e32 1078523331, implicit %exec
+    %3 = V_MAC_F32_e32 %2, killed %0.sub0, killed %1, implicit %exec
+
+...
+
+# GCN-LABEL: name: test_madak_f32
+# GCN: V_MADAK_F32 killed %0.sub0, %0.sub1, 1078523331, implicit %exec
+---
+name:            test_madak_f32
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = V_MOV_B32_e32 1078523331, implicit %exec
+    %2 = V_MAC_F32_e32 killed %0.sub0, %0.sub1, %1, implicit %exec
+
+...
+
+# GCN-LABEL: name: test_madmk_reg_imm_f16
+# GCN: V_MADMK_F16 killed %0.sub0, 1078523331, killed %1, implicit %exec
+---
+name:            test_madmk_reg_imm_f16
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = COPY %0.sub1
+    %2 = V_MOV_B32_e32 1078523331, implicit %exec
+    %3 = V_MAC_F16_e32 killed %0.sub0, %2, killed %1, implicit %exec
+
+...
+
+# GCN-LABEL: name: test_madmk_imm_reg_f16
+# GCN: V_MADMK_F16 killed %0.sub0, 1078523331, killed %1, implicit %exec
+---
+name:            test_madmk_imm_reg_f16
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = COPY %0.sub1
+    %2 = V_MOV_B32_e32 1078523331, implicit %exec
+    %3 = V_MAC_F16_e32 %2, killed %0.sub0, killed %1, implicit %exec
+
+...
+
+# GCN-LABEL: name: test_madak_f16
+# GCN: V_MADAK_F16 killed %0.sub0, %0.sub1, 1078523331, implicit %exec
+---
+name:            test_madak_f16
+registers:
+  - { id: 0, class: vreg_64 }
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = V_MOV_B32_e32 1078523331, implicit %exec
+    %2 = V_MAC_F16_e32 killed %0.sub0, %0.sub1, %1, implicit %exec
+...
+
+# Make sure constant bus restriction isn't violated if src0 is an SGPR.
+
+# GCN-LABEL: name: test_madak_sgpr_src0_f32
+# GCN: %1:vgpr_32 = V_MOV_B32_e32 1078523331, implicit %exec
+# GCN: %2:vgpr_32 = V_MAD_F32 0, killed %0, 0, %1, 0, %3:vgpr_32, 0, 0, implicit %exec
+
+---
+name:            test_madak_sgpr_src0_f32
+registers:
+  - { id: 0, class: sreg_32_xm0 }
+  - { id: 1, class: vgpr_32}
+  - { id: 2, class: vgpr_32 }
+  - { id: 3, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = IMPLICIT_DEF
+    %1 = V_MOV_B32_e32 1078523331, implicit %exec
+    %2 = V_MAC_F32_e32 killed %0, %1, %3, implicit %exec
+
+...
+
+# This can still fold if this is an inline immediate.
+
+# GCN-LABEL: name: test_madak_inlineimm_src0_f32
+# GCN: %1:vgpr_32 = V_MADMK_F32 1073741824, 1078523331, %2:vgpr_32, implicit %exec
+
+---
+name:            test_madak_inlineimm_src0_f32
+registers:
+  - { id: 0, class: vgpr_32}
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = V_MOV_B32_e32 1078523331, implicit %exec
+    %1 = V_MAC_F32_e32 1073741824, %0, %2, implicit %exec
+
+...
+# Non-inline immediate uses constant bus already.
+
+# GCN-LABEL: name: test_madak_otherimm_src0_f32
+# GCN: %1:vgpr_32 = V_MAC_F32_e32 1120403456, %0, %1, implicit %exec
+
+---
+name:            test_madak_otherimm_src0_f32
+registers:
+  - { id: 0, class: vgpr_32}
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+body:             |
+  bb.0:
+
+    %0 = V_MOV_B32_e32 1078523331, implicit %exec
+    %1 = V_MAC_F32_e32 1120403456, %0, %2, implicit %exec
+
+...
+# Non-inline immediate uses constant bus already.
+
+# GCN-LABEL: name: test_madak_other_constantlike_src0_f32
+# GCN: %1:vgpr_32 = V_MAC_F32_e32 %stack.0, %0, %1, implicit %exec
+---
+name:            test_madak_other_constantlike_src0_f32
+registers:
+  - { id: 0, class: vgpr_32}
+  - { id: 1, class: vgpr_32 }
+  - { id: 2, class: vgpr_32 }
+stack:
+  - { id: 0, name: "", type: default, offset: 0, size: 128, alignment: 8,
+      callee-saved-register: '', local-offset: 0, di-variable: '', di-expression: '',
+      di-location: '' }
+body:             |
+  bb.0:
+
+    %0 = V_MOV_B32_e32 1078523331, implicit %exec
+    %1 = V_MAC_F32_e32 %stack.0, %0, %2, implicit %exec
+
+...
diff --git a/test/CodeGen/AMDGPU/uaddo.ll b/test/CodeGen/AMDGPU/uaddo.ll
index 5754bd9bb913a..0cb2487dd4ac6 100644
--- a/test/CodeGen/AMDGPU/uaddo.ll
+++ b/test/CodeGen/AMDGPU/uaddo.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_uaddo_i64_zext:
 ; GCN: s_add_u32
@@ -22,7 +23,10 @@ define amdgpu_kernel void @s_uaddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64
 ; FIXME: Could do scalar
 
 ; FUNC-LABEL: {{^}}s_uaddo_i32:
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG: ADDC_UINT
@@ -37,7 +41,10 @@ define amdgpu_kernel void @s_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_uaddo_i32:
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG: ADDC_UINT
@@ -58,7 +65,10 @@ define amdgpu_kernel void @v_uaddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_uaddo_i32_novcc:
-; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG: ADDC_UINT
@@ -95,8 +105,14 @@ define amdgpu_kernel void @s_uaddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_uaddo_i64:
-; GCN: v_add_i32
-; GCN: v_addc_u32
+; SI: v_add_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_addc_u32_e32 v{{[0-9]+}}, vcc,
+
+; VI: v_add_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_addc_u32_e32 v{{[0-9]+}}, vcc,
+
+; GFX9: v_add_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_addc_co_u32_e32 v{{[0-9]+}}, vcc,
 
 ; EG: ADDC_UINT
 ; EG: ADD_INT
@@ -118,6 +134,9 @@ define amdgpu_kernel void @v_uaddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; FUNC-LABEL: {{^}}v_uaddo_i16:
 ; VI: v_add_u16_e32
 ; VI: v_cmp_lt_u16_e32
+
+; GFX9: v_add_u16_e32
+; GFX9: v_cmp_lt_u16_e32
 define amdgpu_kernel void @v_uaddo_i16(i16 addrspace(1)* %out, i1 addrspace(1)* %carryout, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/udivrem.ll b/test/CodeGen/AMDGPU/udivrem.ll
index 9507a49cfc8b3..08b764824cab4 100644
--- a/test/CodeGen/AMDGPU/udivrem.ll
+++ b/test/CodeGen/AMDGPU/udivrem.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI --check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefix=SI --check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=EG --check-prefix=FUNC %s
 
@@ -30,24 +30,24 @@
 ; SI: v_rcp_iflag_f32_e32 [[RCP:v[0-9]+]]
 ; SI-DAG: v_mul_hi_u32 [[RCP_HI:v[0-9]+]], [[RCP]]
 ; SI-DAG: v_mul_lo_i32 [[RCP_LO:v[0-9]+]], [[RCP]]
-; SI-DAG: v_sub_i32_e32 [[NEG_RCP_LO:v[0-9]+]], vcc, 0, [[RCP_LO]]
+; SI-DAG: v_sub_{{[iu]}}32_e32 [[NEG_RCP_LO:v[0-9]+]], vcc, 0, [[RCP_LO]]
 ; SI: v_cndmask_b32_e64
 ; SI: v_mul_hi_u32 [[E:v[0-9]+]], {{v[0-9]+}}, [[RCP]]
-; SI-DAG: v_add_i32_e32 [[RCP_A_E:v[0-9]+]], vcc, [[E]], [[RCP]]
-; SI-DAG: v_subrev_i32_e32 [[RCP_S_E:v[0-9]+]], vcc, [[E]], [[RCP]]
+; SI-DAG: v_add_{{[iu]}}32_e32 [[RCP_A_E:v[0-9]+]], vcc, [[E]], [[RCP]]
+; SI-DAG: v_subrev_{{[iu]}}32_e32 [[RCP_S_E:v[0-9]+]], vcc, [[E]], [[RCP]]
 ; SI: v_cndmask_b32_e64
 ; SI: v_mul_hi_u32 [[Quotient:v[0-9]+]]
 ; SI: v_mul_lo_i32 [[Num_S_Remainder:v[0-9]+]]
-; SI-DAG: v_add_i32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]]
-; SI-DAG: v_sub_i32_e32 [[Remainder:v[0-9]+]], vcc, {{[vs][0-9]+}}, [[Num_S_Remainder]]
+; SI-DAG: v_add_{{[iu]}}32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]]
+; SI-DAG: v_sub_{{[iu]}}32_e32 [[Remainder:v[0-9]+]], vcc, {{[vs][0-9]+}}, [[Num_S_Remainder]]
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_subrev_i32_e32 [[Quotient_S_One:v[0-9]+]],
-; SI-DAG: v_subrev_i32_e32 [[Remainder_S_Den:v[0-9]+]],
+; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Quotient_S_One:v[0-9]+]],
+; SI-DAG: v_subrev_{{[iu]}}32_e32 [[Remainder_S_Den:v[0-9]+]],
 ; SI: v_and_b32_e32 [[Tmp1:v[0-9]+]]
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32 [[Remainder_A_Den:v[0-9]+]],
+; SI-DAG: v_add_{{[iu]}}32_e32 [[Remainder_A_Den:v[0-9]+]],
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI: s_endpgm
@@ -114,47 +114,47 @@ define amdgpu_kernel void @test_udivrem(i32 addrspace(1)* %out0, i32 addrspace(1
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_and_b32_e32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_and_b32_e32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI: s_endpgm
@@ -264,80 +264,80 @@ define amdgpu_kernel void @test_udivrem_v2(<2 x i32> addrspace(1)* %out, <2 x i3
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_and_b32_e32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_and_b32_e32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_and_b32_e32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_rcp_iflag_f32_e32
 ; SI-DAG: v_mul_hi_u32
 ; SI-DAG: v_mul_lo_i32
-; SI-DAG: v_sub_i32_e32
+; SI-DAG: v_sub_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI-DAG: v_mul_hi_u32
-; SI-DAG: v_add_i32_e32
-; SI-DAG: v_subrev_i32_e32
+; SI-DAG: v_add_{{[iu]}}32_e32
+; SI-DAG: v_subrev_{{[iu]}}32_e32
 ; SI-DAG: v_cndmask_b32_e64
 ; SI: s_endpgm
 define amdgpu_kernel void @test_udivrem_v4(<4 x i32> addrspace(1)* %out, <4 x i32> %x, <4 x i32> %y) {
diff --git a/test/CodeGen/AMDGPU/udivrem64.ll b/test/CodeGen/AMDGPU/udivrem64.ll
index bd297920d5634..408fd01c29222 100644
--- a/test/CodeGen/AMDGPU/udivrem64.ll
+++ b/test/CodeGen/AMDGPU/udivrem64.ll
@@ -1,5 +1,6 @@
 ;RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI --check-prefix=GCN --check-prefix=FUNC %s
 ;RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefix=VI --check-prefix=GCN --check-prefix=FUNC %s
+;RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefix=VI --check-prefix=GCN --check-prefix=FUNC %s
 ;RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=EG --check-prefix=FUNC %s
 
 ;FUNC-LABEL: {{^}}test_udiv:
@@ -36,39 +37,12 @@
 ;EG: BFE_UINT
 ;EG: BFE_UINT
 
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN-NOT: v_mad_f32
-;SI-NOT: v_lshr_b64
-;VI-NOT: v_lshrrev_b64
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0x4f800000,
+;GCN: v_rcp_f32_e32
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x5f7ffffc
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x2f800000
+;GCN: v_trunc_f32_e32
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0xcf800000
 ;GCN: s_endpgm
 define amdgpu_kernel void @test_udiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
   %result = udiv i64 %x, %y
@@ -110,39 +84,12 @@ define amdgpu_kernel void @test_udiv(i64 addrspace(1)* %out, i64 %x, i64 %y) {
 ;EG: BFE_UINT
 ;EG: AND_INT {{.*}}, 1,
 
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN: s_bfe_u32
-;GCN-NOT: v_mad_f32
-;SI-NOT: v_lshr_b64
-;VI-NOT: v_lshrrev_b64
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0x4f800000,
+;GCN: v_rcp_f32_e32
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x5f7ffffc
+;GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x2f800000
+;GCN: v_trunc_f32_e32
+;GCN: v_mac_f32_e32 v{{[0-9]+}}, 0xcf800000
 ;GCN: s_endpgm
 define amdgpu_kernel void @test_urem(i64 addrspace(1)* %out, i64 %x, i64 %y) {
   %result = urem i64 %x, %y
diff --git a/test/CodeGen/AMDGPU/uint_to_fp.i64.ll b/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
index 8d4ac3bd6f144..1dd67ba704e71 100644
--- a/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
+++ b/test/CodeGen/AMDGPU/uint_to_fp.i64.ll
@@ -21,7 +21,7 @@ define amdgpu_kernel void @s_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64
 ; GCN-DAG: v_cmp_eq_u64
 ; GCN-DAG: v_cmp_gt_u64
 
-; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
+; GCN: v_add_{{[iu]}}32_e32 [[VR:v[0-9]+]]
 ; GCN: v_cvt_f16_f32_e32 [[VR_F16:v[0-9]+]], [[VR]]
 ; GCN: {{buffer|flat}}_store_short {{.*}}[[VR_F16]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
@@ -52,7 +52,7 @@ define amdgpu_kernel void @s_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64
 ; GCN-DAG: v_cmp_eq_u64
 ; GCN-DAG: v_cmp_gt_u64
 
-; GCN: v_add_i32_e32 [[VR:v[0-9]+]]
+; GCN: v_add_{{[iu]}}32_e32 [[VR:v[0-9]+]]
 ; GCN: {{buffer|flat}}_store_dword {{.*}}[[VR]]
 define amdgpu_kernel void @v_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* %in) #0 {
   %tid = call i32 @llvm.r600.read.tidig.x()
diff --git a/test/CodeGen/AMDGPU/uitofp.f16.ll b/test/CodeGen/AMDGPU/uitofp.f16.ll
index eaa1d073cafb4..0e7250fadbbbf 100644
--- a/test/CodeGen/AMDGPU/uitofp.f16.ll
+++ b/test/CodeGen/AMDGPU/uitofp.f16.ll
@@ -1,5 +1,5 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}uitofp_i16_to_f16
 ; GCN: buffer_load_ushort v[[A_I16:[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/umed3.ll b/test/CodeGen/AMDGPU/umed3.ll
index 5a579f3575fd1..350be19d6e0d3 100644
--- a/test/CodeGen/AMDGPU/umed3.ll
+++ b/test/CodeGen/AMDGPU/umed3.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SICIVI -check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SICIVI -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx901 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
diff --git a/test/CodeGen/AMDGPU/unaligned-load-store.ll b/test/CodeGen/AMDGPU/unaligned-load-store.ll
index 68aacd084bf94..a08535fc8593d 100644
--- a/test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ b/test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -519,7 +519,7 @@ define amdgpu_kernel void @constant_unaligned_load_v4i32(<4 x i32> addrspace(2)*
 }
 
 ; SI-LABEL: {{^}}constant_align4_load_i8:
-; SI: buffer_load_ubyte
+; SI: s_load_dword
 ; SI: buffer_store_byte
 define amdgpu_kernel void @constant_align4_load_i8(i8 addrspace(2)* %p, i8 addrspace(1)* %r) #0 {
   %v = load i8, i8 addrspace(2)* %p, align 4
diff --git a/test/CodeGen/AMDGPU/undefined-physreg-sgpr-spill.mir b/test/CodeGen/AMDGPU/undefined-physreg-sgpr-spill.mir
new file mode 100644
index 0000000000000..7c445b7ca4e6b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/undefined-physreg-sgpr-spill.mir
@@ -0,0 +1,141 @@
+# RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 -run-pass si-optimize-exec-masking -verify-machineinstrs -o -  %s | FileCheck %s
+--- |
+  define amdgpu_kernel void @undefined_physreg_sgpr_spill() #0 {
+    unreachable
+  }
+
+  define amdgpu_kernel void @undefined_physreg_sgpr_spill_reorder() #0 {
+    unreachable
+  }
+
+  attributes #0 = { nounwind "amdgpu-num-sgpr"="16" }
+
+...
+---
+
+# copy + s_and_b64 was turned into saveexec, deleting the copy,
+# leaving a spill of the undefined register.
+
+# CHECK-LABEL: name: undefined_physreg_sgpr_spill
+# CHECK: %sgpr0_sgpr1 = COPY %exec, implicit-def %exec
+# CHECK-NEXT: SI_SPILL_S64_SAVE %sgpr0_sgpr1,
+# CHECK-NEXT: %sgpr2_sgpr3 = S_AND_B64 killed %sgpr0_sgpr1, killed %vcc, implicit-def dead %scc
+# CHECK: %exec = COPY killed %sgpr2_sgpr3
+name:            undefined_physreg_sgpr_spill
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '' }
+  - { reg: '%sgpr4_sgpr5', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 8, alignment: 4,
+      stack-id: 1, callee-saved-register: '', callee-saved-restored: true,
+      di-variable: '', di-expression: '', di-location: '' }
+constants:
+body:             |
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: %vgpr0, %sgpr4_sgpr5
+
+    %vgpr1_vgpr2 = COPY killed %sgpr4_sgpr5, implicit %exec
+    %vgpr1 = GLOBAL_LOAD_UBYTE killed %vgpr1_vgpr2, 0, 0, 0, implicit %exec :: (non-temporal dereferenceable invariant load 1 from `i1 addrspace(2)* undef`)
+    %vcc = V_CMP_NE_U32_e64 0, %vgpr0, implicit %exec
+    %sgpr0_sgpr1 = V_CMP_EQ_U32_e64 1, killed %vgpr1, implicit %exec
+    %vgpr1 = V_CNDMASK_B32_e64 0, -1, killed %sgpr0_sgpr1, implicit %exec
+    %sgpr0_sgpr1 = COPY %exec, implicit-def %exec
+    SI_SPILL_S64_SAVE %sgpr0_sgpr1, %stack.0, implicit %exec, implicit %sgpr8_sgpr9_sgpr10_sgpr11, implicit %sgpr13, implicit-def dead %m0 :: (store 8 into %stack.0, align 4)
+    %sgpr2_sgpr3 = S_AND_B64 killed %sgpr0_sgpr1, killed %vcc, implicit-def dead %scc
+    %exec = S_MOV_B64_term killed %sgpr2_sgpr3
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+    liveins: %vgpr0, %vgpr1
+
+    %sgpr2_sgpr3 = S_MOV_B64 0
+    %vgpr2 = V_MOV_B32_e32 0, implicit %exec
+    %sgpr4_sgpr5 = IMPLICIT_DEF
+    S_BRANCH %bb.3
+
+  bb.2:
+    successors:
+
+    %sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.0, implicit %exec, implicit %sgpr8_sgpr9_sgpr10_sgpr11, implicit %sgpr13, implicit-def dead %m0 :: (load 8 from %stack.0, align 4)
+    %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc
+
+  bb.3:
+    liveins: %vgpr0, %vgpr1, %vgpr2, %sgpr2_sgpr3, %sgpr4_sgpr5
+
+    %vcc = COPY %vgpr1
+    S_ENDPGM
+
+...
+---
+
+# Move spill to after future save instruction
+# CHECK-LABEL: {{^}}name: undefined_physreg_sgpr_spill_reorder
+# CHECK: %sgpr0_sgpr1 = COPY %exec, implicit-def %exec
+# CHECK: %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def dead %scc
+# CHECK: SI_SPILL_S64_SAVE killed %sgpr0_sgpr1, %stack.0, implicit %exec, implicit %sgpr8_sgpr9_sgpr10_sgpr11, implicit %sgpr13, implicit-def dead %m0 :: (store 8 into %stack.0, align 4)
+# CHECK: %exec = COPY killed %sgpr2_sgpr3
+name:            undefined_physreg_sgpr_spill_reorder
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+liveins:
+  - { reg: '%vgpr0', virtual-reg: '' }
+  - { reg: '%sgpr4_sgpr5', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 8, alignment: 4,
+      stack-id: 1, callee-saved-register: '', callee-saved-restored: true,
+      di-variable: '', di-expression: '', di-location: '' }
+constants:
+body:             |
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: %vgpr0, %sgpr4_sgpr5
+
+    %vgpr1_vgpr2 = COPY killed %sgpr4_sgpr5, implicit %exec
+    %vgpr1 = GLOBAL_LOAD_UBYTE killed %vgpr1_vgpr2, 0, 0, 0, implicit %exec :: (non-temporal dereferenceable invariant load 1 from `i1 addrspace(2)* undef`)
+    %vcc = V_CMP_NE_U32_e64 0, %vgpr0, implicit %exec
+    %sgpr0_sgpr1 = V_CMP_EQ_U32_e64 1, killed %vgpr1, implicit %exec
+    %vgpr1 = V_CNDMASK_B32_e64 0, -1, killed %sgpr0_sgpr1, implicit %exec
+    %sgpr0_sgpr1 = COPY %exec, implicit-def %exec
+    %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def dead %scc
+    SI_SPILL_S64_SAVE killed %sgpr0_sgpr1, %stack.0, implicit %exec, implicit %sgpr8_sgpr9_sgpr10_sgpr11, implicit %sgpr13, implicit-def dead %m0 :: (store 8 into %stack.0, align 4)
+    %exec = S_MOV_B64_term killed %sgpr2_sgpr3
+    SI_MASK_BRANCH %bb.2, implicit %exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+    liveins: %vgpr0, %vgpr1
+
+    %sgpr2_sgpr3 = S_MOV_B64 0
+    %vgpr2 = V_MOV_B32_e32 0, implicit %exec
+    %sgpr4_sgpr5 = IMPLICIT_DEF
+    S_BRANCH %bb.3
+
+  bb.2:
+    successors:
+
+    %sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.0, implicit %exec, implicit %sgpr8_sgpr9_sgpr10_sgpr11, implicit %sgpr13, implicit-def dead %m0 :: (load 8 from %stack.0, align 4)
+    %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc
+
+  bb.3:
+    liveins: %vgpr0, %vgpr1, %vgpr2, %sgpr2_sgpr3, %sgpr4_sgpr5
+
+    %vcc = COPY %vgpr1
+    S_ENDPGM
+
+...
diff --git a/test/CodeGen/AMDGPU/uniform-PHI.ll b/test/CodeGen/AMDGPU/uniform-PHI.ll
new file mode 100644
index 0000000000000..3cb86b39a65f2
--- /dev/null
+++ b/test/CodeGen/AMDGPU/uniform-PHI.ll
@@ -0,0 +1,39 @@
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: BB0_2
+; GCN-NOT: v_readfirstlane
+
+
+target triple = "amdgcn--amdhsa"
+define amdgpu_kernel void @uniform-PHI(i32 addrspace(1)* nocapture readonly %arg, i32 addrspace(1)* nocapture %arg1, i32 %arg2) {
+bb:
+  %tmp = sext i32 %arg2 to i64
+  %tmp3 = tail call i64 @_Z13get_global_idj(i32 0) #2
+  %tmp4 = icmp ugt i64 %tmp3, %tmp
+  %tmp5 = icmp sgt i32 %arg2, 0
+  %tmp6 = and i1 %tmp4, %tmp5
+  br i1 %tmp6, label %bb7, label %bb17
+
+bb7:                                              ; preds = %bb
+  br label %bb8
+
+bb8:                                              ; preds = %bb8, %bb7
+  %tmp9 = phi i32 [ %tmp15, %bb8 ], [ 0, %bb7 ]
+  %tmp10 = phi i32 [ %tmp14, %bb8 ], [ 0, %bb7 ]
+  %tmp11 = zext i32 %tmp9 to i64
+  %tmp12 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp11
+  %tmp13 = load i32, i32 addrspace(1)* %tmp12, align 4
+  %tmp14 = add nsw i32 %tmp13, %tmp10
+  %tmp15 = add nuw nsw i32 %tmp9, 1
+  %tmp16 = icmp eq i32 %tmp15, %arg2
+  br i1 %tmp16, label %bb17, label %bb8
+
+bb17:                                             ; preds = %bb8, %bb
+  %tmp18 = phi i32 [ 0, %bb ], [ %tmp14, %bb8 ]
+  store i32 %tmp18, i32 addrspace(1)* %arg1, align 4
+  ret void
+}
+
+declare i64 @_Z13get_global_idj(i32) local_unnamed_addr #1
+attributes #1 = { convergent nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="fiji" "target-features"="+16-bit-insts,+dpp,+fp64-fp16-denormals,+s-memrealtime,-fp32-denormals" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { convergent nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/uniform-cfg.ll b/test/CodeGen/AMDGPU/uniform-cfg.ll
index a9d45d71fa2ea..a247d7a343fbc 100644
--- a/test/CodeGen/AMDGPU/uniform-cfg.ll
+++ b/test/CodeGen/AMDGPU/uniform-cfg.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=verde -amdgpu-early-ifcvt=0 -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-early-ifcvt=0 -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=verde -amdgpu-early-ifcvt=0 -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-early-ifcvt=0 -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}uniform_if_scc:
 ; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0
@@ -303,7 +303,6 @@ done:
 ; GCN-LABEL: {{^}}uniform_inside_divergent:
 ; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}}
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN: s_xor_b64  [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]]
 ; GCN: s_cmp_lg_u32 {{s[0-9]+}}, 0
 ; GCN: s_cbranch_scc0 [[IF_UNIFORM_LABEL:[A-Z0-9_a-z]+]]
 ; GCN: s_endpgm
@@ -331,13 +330,14 @@ endif:
 
 ; GCN-LABEL: {{^}}divergent_inside_uniform:
 ; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0
-; GCN: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]]
-; GCN: [[IF_LABEL]]:
+; GCN: s_cbranch_scc1 [[ENDIF_LABEL:[0-9_A-Za-z]+]]
 ; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}}
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN: s_xor_b64  [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]]
+; GCN: ; mask branch [[ENDIF_LABEL]]
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
+; GCN: [[ENDIF_LABEL]]:
+; GCN: s_endpgm
 define amdgpu_kernel void @divergent_inside_uniform(i32 addrspace(1)* %out, i32 %cond) {
 entry:
   %u_cmp = icmp eq i32 %cond, 0
@@ -360,7 +360,6 @@ endif:
 ; GCN-LABEL: {{^}}divergent_if_uniform_if:
 ; GCN: v_cmp_eq_u32_e32 vcc, 0, v0
 ; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc
-; GCN: s_xor_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]]
 ; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1
 ; GCN: buffer_store_dword [[ONE]]
 ; GCN: s_or_b64 exec, exec, [[MASK]]
@@ -402,7 +401,7 @@ exit:
 ; GCN: s_cmp_lt_i32 [[COND]], 1
 ; GCN: s_cbranch_scc1 BB[[FNNUM:[0-9]+]]_3
 
-; GCN: BB#1:
+; GCN: %bb.1:
 ; GCN-NOT: cmp
 ; GCN: buffer_load_dword
 ; GCN: buffer_store_dword
@@ -503,7 +502,7 @@ done:
 ; GCN: s_mov_b32 [[S_VAL]], 1
 
 ; GCN: [[IF_LABEL]]:
-; GCN: v_mov_b32_e32 [[V_VAL]], [[S_VAL]]
+; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]]
 ; GCN: buffer_store_dword [[V_VAL]]
 define amdgpu_kernel void @uniform_if_scc_i64_sgt(i64 %cond, i32 addrspace(1)* %out) {
 entry:
@@ -560,6 +559,28 @@ done:
   ret void
 }
 
+; GCN-LABEL: {{^}}move_to_valu_vgpr_operand_phi:
+; GCN: v_add_{{[iu]}}32_e32
+; GCN: ds_write_b32
+define void @move_to_valu_vgpr_operand_phi(i32 addrspace(3)* %out) {
+bb0:
+  br label %bb1
+
+bb1:                                              ; preds = %bb3, %bb0
+  %tmp0 = phi i32 [ 8, %bb0 ], [ %tmp4, %bb3 ]
+  %tmp1 = add nsw i32 %tmp0, -1
+  %tmp2 = getelementptr inbounds i32, i32 addrspace(3)* %out, i32 %tmp1
+  br i1 undef, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  store volatile i32 1, i32 addrspace(3)* %tmp2, align 4
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb1
+  %tmp4 = add nsw i32 %tmp0, 2
+  br label %bb1
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 
 attributes #0 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll b/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
index 8a08f9d8bb0d7..1bbda66fddb16 100644
--- a/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
+++ b/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll
@@ -5,13 +5,11 @@
 ; CHECK-LABEL: {{^}}test1:
 ; CHECK: v_cmp_ne_u32_e32 vcc, 0
 ; CHECK: s_and_saveexec_b64
-; CHECK-NEXT: s_xor_b64
 ; CHECK-NEXT: ; mask branch
-; CHECK-NEXT: s_cbranch_execz BB{{[0-9]+_[0-9]+}}
 ; CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %loop_body.preheader
 
 ; CHECK: [[LOOP_BODY_LABEL:BB[0-9]+_[0-9]+]]:
-; CHECK: s_cbranch_vccz [[LOOP_BODY_LABEL]]
+; CHECK: s_cbranch_scc0 [[LOOP_BODY_LABEL]]
 
 ; CHECK: s_endpgm
 define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <2 x i32> %addr.base, i32 %y, i32 %p) {
@@ -35,9 +33,7 @@ out:
 
 ; CHECK-LABEL: {{^}}test2:
 ; CHECK: s_and_saveexec_b64
-; CHECK-NEXT: s_xor_b64
 ; CHECK-NEXT: ; mask branch
-; CHECK-NEXT: s_cbranch_execz
 define amdgpu_kernel void @test2(i32 addrspace(1)* %out, i32 %a, i32 %b) {
 main_body:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/test/CodeGen/AMDGPU/unknown-processor.ll b/test/CodeGen/AMDGPU/unknown-processor.ll
index e25f2235993f5..6dfcff77d8130 100644
--- a/test/CodeGen/AMDGPU/unknown-processor.ll
+++ b/test/CodeGen/AMDGPU/unknown-processor.ll
@@ -1,5 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=GCN %s
-; RUN: llc -march=r600 -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=R600 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=GCN %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=unknown -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR -check-prefix=R600 %s
+target datalayout = "A5"
 
 ; Should not crash when the processor is not recognized and the
 ; wavefront size feature not set.
@@ -14,7 +15,7 @@
 
 ; R600: MOV
 define amdgpu_kernel void @foo() {
-  %alloca = alloca i32, align 4
-  store volatile i32 0, i32* %alloca
+  %alloca = alloca i32, align 4, addrspace(5)
+  store volatile i32 0, i32 addrspace(5)* %alloca
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/unsupported-calls.ll b/test/CodeGen/AMDGPU/unsupported-calls.ll
new file mode 100644
index 0000000000000..68872c54f7fb0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/unsupported-calls.ll
@@ -0,0 +1,69 @@
+; RUN: not llc -march=amdgcn -mtriple=amdgcn---amdgiz -tailcallopt < %s 2>&1 | FileCheck -check-prefix=GCN %s
+; RUN: not llc -march=r600 -mtriple=r600---amdgiz -mcpu=cypress -tailcallopt < %s 2>&1 | FileCheck -check-prefix=R600 %s
+
+declare i32 @external_function(i32) nounwind
+
+; GCN-NOT: error
+; R600: in function test_call_external{{.*}}: unsupported call to function external_function
+define amdgpu_kernel void @test_call_external(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %a = load i32, i32 addrspace(1)* %in
+  %b = load i32, i32 addrspace(1)* %b_ptr
+  %c = call i32 @external_function(i32 %b) nounwind
+  %result = add i32 %a, %c
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
+define i32 @defined_function(i32 %x) nounwind noinline {
+  %y = add i32 %x, 8
+  ret i32 %y
+}
+
+; GCN-NOT: error
+; R600: in function test_call{{.*}}: unsupported call to function defined_function
+define amdgpu_kernel void @test_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %a = load i32, i32 addrspace(1)* %in
+  %b = load i32, i32 addrspace(1)* %b_ptr
+  %c = call i32 @defined_function(i32 %b) nounwind
+  %result = add i32 %a, %c
+  store i32 %result, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN: error: <unknown>:0:0: in function test_tail_call i32 (i32 addrspace(1)*, i32 addrspace(1)*): unsupported required tail call to function defined_function
+; R600: in function test_tail_call{{.*}}: unsupported call to function defined_function
+define i32 @test_tail_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
+  %b_ptr = getelementptr i32, i32 addrspace(1)* %in, i32 1
+  %a = load i32, i32 addrspace(1)* %in
+  %b = load i32, i32 addrspace(1)* %b_ptr
+  %c = tail call i32 @defined_function(i32 %b)
+  ret i32 %c
+}
+
+declare void @external.varargs(i32, double, i64, ...)
+
+; GCN: error: <unknown>:0:0: in function test_call_varargs void (): unsupported call to variadic function external.varargs
+; R600: in function test_call_varargs{{.*}}: unsupported call to function external.varargs
+define void @test_call_varargs() {
+  call void (i32, double, i64, ...) @external.varargs(i32 42, double 1.0, i64 12, i8 3, i16 1, i32 4, float 1.0, double 2.0)
+  ret void
+}
+
+declare i32 @extern_variadic(...)
+
+; GCN: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported indirect call to function extern_variadic
+; R600: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported call to function extern_variadic
+define i32 @test_tail_call_bitcast_extern_variadic(<4 x float> %arg0, <4 x float> %arg1, i32 %arg2) {
+  %add = fadd <4 x float> %arg0, %arg1
+  %call = tail call i32 bitcast (i32 (...)* @extern_variadic to i32 (<4 x float>)*)(<4 x float> %add)
+  ret i32 %call
+}
+
+; GCN: :0:0: in function test_indirect_call void (void ()*): unsupported indirect call to function <unknown>
+; R600: in function test_indirect_call{{.*}}: unsupported call to function <unknown>
+define void @test_indirect_call(void()* %fptr) {
+  call void %fptr()
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/urem.ll b/test/CodeGen/AMDGPU/urem.ll
index 823c918dcda70..35d5735f87bff 100644
--- a/test/CodeGen/AMDGPU/urem.ll
+++ b/test/CodeGen/AMDGPU/urem.ll
@@ -21,9 +21,9 @@ define amdgpu_kernel void @test_urem_i32(i32 addrspace(1)* %out, i32 addrspace(1
 ; FUNC-LABEL: {{^}}test_urem_i32_7:
 ; SI: v_mov_b32_e32 [[MAGIC:v[0-9]+]], 0x24924925
 ; SI: v_mul_hi_u32 [[MAGIC]], {{v[0-9]+}}
-; SI: v_subrev_i32
+; SI: v_subrev_{{[iu]}}32
 ; SI: v_mul_lo_i32
-; SI: v_sub_i32
+; SI: v_sub_{{[iu]}}32
 ; SI: buffer_store_dword
 ; SI: s_endpgm
 define amdgpu_kernel void @test_urem_i32_7(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
diff --git a/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll b/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
index e6bdb68a4f775..3cce2c96e8380 100644
--- a/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
+++ b/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
@@ -45,14 +45,16 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_a_b(float addrspace(
 ; GCN-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}}
 ; GCN-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xc|0x30}}
 ; SI: buffer_load_dword [[VA0:v[0-9]+]]
-; SI: buffer_load_dword [[VA1:v[0-9]+]]
+; SI-NEXT: buffer_load_dword [[VA1:v[0-9]+]]
 
 ; GCN-NOT: v_mov_b32
-; GCN: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]]
-; GCN-NOT: v_mov_b32
 
 ; VI: buffer_load_dword [[VA0:v[0-9]+]]
-; VI: buffer_load_dword [[VA1:v[0-9]+]]
+; VI-NEXT: buffer_load_dword [[VA1:v[0-9]+]]
+
+; GCN-NOT: v_mov_b32
+; GCN: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]]
+; GCN-NOT: v_mov_b32
 
 ; GCN-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[SA]], [[VA0]], [[VB]]
 ; GCN-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[SA]], [[VA1]], [[VB]]
diff --git a/test/CodeGen/AMDGPU/usubo.ll b/test/CodeGen/AMDGPU/usubo.ll
index f01bf498e0d8a..eeb19f86f3849 100644
--- a/test/CodeGen/AMDGPU/usubo.ll
+++ b/test/CodeGen/AMDGPU/usubo.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_usubo_i64_zext:
 ; GCN: s_sub_u32
@@ -22,7 +23,10 @@ define amdgpu_kernel void @s_usubo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64
 ; FIXME: Could do scalar
 
 ; FUNC-LABEL: {{^}}s_usubo_i32:
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG-DAG: SUBB_UINT
@@ -37,7 +41,10 @@ define amdgpu_kernel void @s_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_usubo_i32:
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG-DAG: SUBB_UINT
@@ -58,7 +65,10 @@ define amdgpu_kernel void @v_usubo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_usubo_i32_novcc:
-; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+
 ; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
 
 ; EG-DAG: SUBB_UINT
@@ -97,8 +107,13 @@ define amdgpu_kernel void @s_usubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_usubo_i64:
-; GCN: v_sub_i32
-; GCN: v_subb_u32
+; SI: v_sub_i32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; SI: v_subb_u32
+; VI: v_sub_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; VI: v_subb_u32
+
+; GFX9: v_sub_co_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}
+; GFX9: v_subb_co_u32
 
 ; EG-DAG: SUBB_UINT
 ; EG-DAG: SUB_INT
@@ -120,8 +135,15 @@ define amdgpu_kernel void @v_usubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 }
 
 ; FUNC-LABEL: {{^}}v_usubo_i16:
+; SI: v_subrev_i32_e32
+; SI: v_and_b32
+; SI: v_cmp_ne_u32_e32
+
 ; VI: v_sub_u16_e32
 ; VI: v_cmp_gt_u16_e32
+
+; GFX9: v_sub_u16_e32
+; GFX9: v_cmp_gt_u16_e32
 define amdgpu_kernel void @v_usubo_i16(i16 addrspace(1)* %out, i1 addrspace(1)* %carryout, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/v_cndmask.ll b/test/CodeGen/AMDGPU/v_cndmask.ll
index 5cbfae34e1bb5..bd7738e584649 100644
--- a/test/CodeGen/AMDGPU/v_cndmask.ll
+++ b/test/CodeGen/AMDGPU/v_cndmask.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 
@@ -60,8 +60,8 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprZ_f32(float addrspace(1)*
 ; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k1_sgprX_f32:
 ; GCN: s_load_dword [[X:s[0-9]+]]
 ; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0
-; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]]
-; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[VZ]], vcc
+; GCN-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[X]]
+; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[VX]], vcc
 define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprX_f32(float addrspace(1)* %out, float %x) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -73,7 +73,8 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprX_f32(float addrspace(1)*
 }
 
 ; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k0_sgprZ_f32:
-; GCN: s_load_dword [[X:s[0-9]+]]
+; GCN-DAG: s_load_dword [[X:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}}
+; GCN-DAG: s_load_dword [[Z:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xc|0x30}}
 ; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0
 ; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]]
 ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[VZ]], vcc
@@ -90,8 +91,8 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprZ_f32(float addrspace(1)*
 ; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k0_sgprX_f32:
 ; GCN: s_load_dword [[X:s[0-9]+]]
 ; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0
-; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]]
-; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[VZ]], vcc
+; GCN-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[X]]
+; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[VX]], vcc
 define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprX_f32(float addrspace(1)* %out, float %x) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
@@ -122,7 +123,7 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(float addrspace(1)*
 ; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k1_vgprZ_f32:
 ; GCN-DAG: {{buffer|flat}}_load_dword [[Z:v[0-9]+]]
 ; GCN-DAG: s_load_dword [[X:s[0-9]+]]
-; GCN: v_cmp_nlg_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], [[X]], 0
+; GCN-DAG: v_cmp_nlg_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], [[X]], 0
 ; GCN: v_cndmask_b32_e{{32|64}} v{{[0-9]+}}, 1.0, [[Z]], [[COND]]
 define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_vgprZ_f32(float addrspace(1)* %out, float %x, float addrspace(1)* %z.ptr) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/test/CodeGen/AMDGPU/v_mac_f16.ll b/test/CodeGen/AMDGPU/v_mac_f16.ll
index 46c9b7ee1a3d5..fe5e5a18ab52e 100644
--- a/test/CodeGen/AMDGPU/v_mac_f16.ll
+++ b/test/CodeGen/AMDGPU/v_mac_f16.ll
@@ -1,4 +1,4 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tahiti -mattr=-fp64-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=fiji -mattr=-fp64-fp16-denormals,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
 
 ; GCN-LABEL: {{^}}mac_f16:
diff --git a/test/CodeGen/AMDGPU/v_madak_f16.ll b/test/CodeGen/AMDGPU/v_madak_f16.ll
index 0148ff470b783..ab47cc99a216e 100644
--- a/test/CodeGen/AMDGPU/v_madak_f16.ll
+++ b/test/CodeGen/AMDGPU/v_madak_f16.ll
@@ -23,9 +23,9 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}madak_f16_use_2
-; SI:  v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; SI:  v_madak_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, 0x41200000
 ; SI:  v_mac_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; VI:  v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
+; VI:  v_madak_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, 0x4900
 ; VI:  v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; GCN: s_endpgm
 define amdgpu_kernel void @madak_f16_use_2(
diff --git a/test/CodeGen/AMDGPU/valu-i1.ll b/test/CodeGen/AMDGPU/valu-i1.ll
index a67f36d0a7e8a..4a3937e44f36f 100644
--- a/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/test/CodeGen/AMDGPU/valu-i1.ll
@@ -18,7 +18,6 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 ; SI-NOT: s_mov_b64 s[{{[0-9]:[0-9]}}], -1
 ; SI: v_mov_b32_e32 v{{[0-9]}}, -1
 ; SI: s_and_saveexec_b64
-; SI-NEXT: s_xor_b64
 ; SI-NEXT: ; mask branch
 
 ; v_mov should be after exec modification
@@ -66,14 +65,12 @@ end:
 ; SI-LABEL: {{^}}simple_test_v_if:
 ; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
 ; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
-; SI: ; mask branch [[EXIT:BB[0-9]+_[0-9]+]]
+; SI-NEXT: ; mask branch [[EXIT:BB[0-9]+_[0-9]+]]
 
 ; SI-NEXT: BB{{[0-9]+_[0-9]+}}:
 ; SI: buffer_store_dword
 
 ; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_or_b64 exec, exec, [[BR_SREG]]
 ; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if(i32 addrspace(1)* %dst, i32 addrspace(1)* %src) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
@@ -94,14 +91,12 @@ exit:
 ; SI-LABEL: {{^}}simple_test_v_if_ret_else_ret:
 ; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
 ; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
-; SI: ; mask branch [[EXIT:BB[0-9]+_[0-9]+]]
+; SI-NEXT: ; mask branch [[EXIT:BB[0-9]+_[0-9]+]]
 
 ; SI-NEXT: BB{{[0-9]+_[0-9]+}}:
 ; SI: buffer_store_dword
 
 ; SI-NEXT: {{^}}[[EXIT]]:
-; SI: s_or_b64 exec, exec, [[BR_SREG]]
 ; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if_ret_else_ret(i32 addrspace(1)* %dst, i32 addrspace(1)* %src) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -140,7 +135,6 @@ exit:
 ; SI-NEXT: buffer_store_dword
 
 ; SI-NEXT: {{^}}[[UNIFIED_RETURN]]: ; %UnifiedReturnBlock
-; SI: s_or_b64 exec, exec
 ; SI: s_endpgm
 define amdgpu_kernel void @simple_test_v_if_ret_else_code_ret(i32 addrspace(1)* %dst, i32 addrspace(1)* %src) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -160,16 +154,16 @@ exit:
 ; SI-LABEL: {{^}}simple_test_v_loop:
 ; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}}
 ; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]]
-; SI: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]]
+; SI-NEXT: ; mask branch
+; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]]
 
 ; SI: s_mov_b64 {{s\[[0-9]+:[0-9]+\]}}, 0{{$}}
 
 ; SI: [[LABEL_LOOP:BB[0-9]+_[0-9]+]]:
 ; SI: buffer_load_dword
 ; SI-DAG: buffer_store_dword
-; SI-DAG: v_cmp_eq_u32_e32 vcc, 0x100
-; SI: s_cbranch_vccz [[LABEL_LOOP]]
+; SI-DAG: s_cmpk_eq_i32 s{{[0-9]+}}, 0x100
+; SI: s_cbranch_scc0 [[LABEL_LOOP]]
 ; SI: [[LABEL_EXIT]]:
 ; SI: s_endpgm
 
@@ -198,12 +192,12 @@ exit:
 
 ; Load loop limit from buffer
 ; Branch to exit if uniformly not taken
-; SI: ; BB#0:
+; SI: ; %bb.0:
 ; SI: buffer_load_dword [[VBOUND:v[0-9]+]]
 ; SI: v_cmp_lt_i32_e32 vcc
 ; SI: s_and_saveexec_b64 [[OUTER_CMP_SREG:s\[[0-9]+:[0-9]+\]]], vcc
-; SI: s_xor_b64 [[OUTER_CMP_SREG]], exec, [[OUTER_CMP_SREG]]
-; SI: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]]
+; SI-NEXT: ; mask branch
+; SI-NEXT: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]]
 
 ; Initialize inner condition to false
 ; SI: BB{{[0-9]+_[0-9]+}}: ; %bb10.preheader
@@ -233,9 +227,6 @@ exit:
 ; SI-NEXT: s_andn2_b64 exec, exec, [[COND_STATE]]
 ; SI-NEXT: s_cbranch_execnz [[LABEL_LOOP]]
 
-; SI: BB#5
-; SI: s_or_b64 exec, exec, [[COND_STATE]]
-
 ; SI: [[LABEL_EXIT]]:
 ; SI-NOT: [[COND_STATE]]
 ; SI: s_endpgm
diff --git a/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir b/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
index 54991d3d953cd..ff9826baf48c4 100644
--- a/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
+++ b/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
@@ -48,7 +48,7 @@
 # CHECK-LABEL: name: vccz_corrupt_workaround
 # CHECK: %vcc = V_CMP_EQ_F32
 # CHECK-NEXT: %vcc = S_MOV_B64 %vcc
-# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2.else, implicit killed %vcc
+# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2, implicit killed %vcc
 
 name:            vccz_corrupt_workaround
 alignment:       0
@@ -82,7 +82,7 @@ body:             |
     %sgpr7 = S_MOV_B32 61440
     %sgpr6 = S_MOV_B32 -1
     %vcc = V_CMP_EQ_F32_e64 0, 0, 0, %sgpr2, 0, implicit %exec
-    S_CBRANCH_VCCZ %bb.1.else, implicit killed %vcc
+    S_CBRANCH_VCCZ %bb.1, implicit killed %vcc
 
   bb.2.if:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
@@ -90,7 +90,7 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 9, implicit %exec
     BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     %vgpr0 = V_MOV_B32_e32 0, implicit %exec
-    S_BRANCH %bb.3.done
+    S_BRANCH %bb.3
 
   bb.1.else:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
@@ -111,7 +111,7 @@ body:             |
 ---
 # CHECK-LABEL: name: vccz_corrupt_undef_vcc
 # CHECK: S_WAITCNT
-# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2.else, implicit undef %vcc
+# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2, implicit undef %vcc
 
 name:            vccz_corrupt_undef_vcc
 alignment:       0
@@ -143,7 +143,7 @@ body:             |
     %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`)
     %sgpr7 = S_MOV_B32 61440
     %sgpr6 = S_MOV_B32 -1
-    S_CBRANCH_VCCZ %bb.1.else, implicit undef %vcc
+    S_CBRANCH_VCCZ %bb.1, implicit undef %vcc
 
   bb.2.if:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
@@ -151,7 +151,7 @@ body:             |
     %vgpr0 = V_MOV_B32_e32 9, implicit %exec
     BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     %vgpr0 = V_MOV_B32_e32 0, implicit %exec
-    S_BRANCH %bb.3.done
+    S_BRANCH %bb.3
 
   bb.1.else:
     liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
diff --git a/test/CodeGen/AMDGPU/vector-alloca.ll b/test/CodeGen/AMDGPU/vector-alloca.ll
index a0aac8c1d9ba5..fbdfe48cd3896 100644
--- a/test/CodeGen/AMDGPU/vector-alloca.ll
+++ b/test/CodeGen/AMDGPU/vector-alloca.ll
@@ -1,9 +1,10 @@
-; RUN: llc -march=amdgcn -mcpu=verde -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=verde -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck --check-prefix=EG -check-prefix=FUNC %s
-; RUN: opt -S -mtriple=amdgcn-- -amdgpu-promote-alloca -sroa -instcombine < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=verde -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mtriple=r600---amdgiz -mcpu=redwood < %s | FileCheck --check-prefix=EG -check-prefix=FUNC %s
+; RUN: opt -S -mtriple=amdgcn---amdgiz -amdgpu-promote-alloca -sroa -instcombine < %s | FileCheck -check-prefix=OPT %s
+target datalayout = "A5"
 
 ; OPT-LABEL: @vector_read(
 ; OPT: %0 = extractelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 %index
@@ -17,17 +18,17 @@
 ; EG: MOVA_INT
 define amdgpu_kernel void @vector_read(i32 addrspace(1)* %out, i32 %index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0
-  %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2
-  %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3
-  store i32 0, i32* %x
-  store i32 1, i32* %y
-  store i32 2, i32* %z
-  store i32 3, i32* %w
-  %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %index
-  %tmp2 = load i32, i32* %tmp1
+  %tmp = alloca [4 x i32], addrspace(5)
+  %x = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %y = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %z = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2
+  %w = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 3
+  store i32 0, i32 addrspace(5)* %x
+  store i32 1, i32 addrspace(5)* %y
+  store i32 2, i32 addrspace(5)* %z
+  store i32 3, i32 addrspace(5)* %w
+  %tmp1 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i32, i32 addrspace(5)* %tmp1
   store i32 %tmp2, i32 addrspace(1)* %out
   ret void
 }
@@ -46,19 +47,19 @@ entry:
 ; EG: MOVA_INT
 define amdgpu_kernel void @vector_write(i32 addrspace(1)* %out, i32 %w_index, i32 %r_index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0
-  %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2
-  %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3
-  store i32 0, i32* %x
-  store i32 0, i32* %y
-  store i32 0, i32* %z
-  store i32 0, i32* %w
-  %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %w_index
-  store i32 1, i32* %tmp1
-  %tmp2 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %r_index
-  %tmp3 = load i32, i32* %tmp2
+  %tmp = alloca [4 x i32], addrspace(5)
+  %x = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %y = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %z = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2
+  %w = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 3
+  store i32 0, i32 addrspace(5)* %x
+  store i32 0, i32 addrspace(5)* %y
+  store i32 0, i32 addrspace(5)* %z
+  store i32 0, i32 addrspace(5)* %w
+  %tmp1 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 %w_index
+  store i32 1, i32 addrspace(5)* %tmp1
+  %tmp2 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 %r_index
+  %tmp3 = load i32, i32 addrspace(5)* %tmp2
   store i32 %tmp3, i32 addrspace(1)* %out
   ret void
 }
@@ -73,19 +74,19 @@ entry:
 ; EG: STORE_RAW
 define amdgpu_kernel void @bitcast_gep(i32 addrspace(1)* %out, i32 %w_index, i32 %r_index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0
-  %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2
-  %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3
-  store i32 0, i32* %x
-  store i32 0, i32* %y
-  store i32 0, i32* %z
-  store i32 0, i32* %w
-  %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %tmp2 = bitcast i32* %tmp1 to [4 x i32]*
-  %tmp3 = getelementptr [4 x i32], [4 x i32]* %tmp2, i32 0, i32 0
-  %tmp4 = load i32, i32* %tmp3
+  %tmp = alloca [4 x i32], addrspace(5)
+  %x = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %y = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %z = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2
+  %w = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 3
+  store i32 0, i32 addrspace(5)* %x
+  store i32 0, i32 addrspace(5)* %y
+  store i32 0, i32 addrspace(5)* %z
+  store i32 0, i32 addrspace(5)* %w
+  %tmp1 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %tmp2 = bitcast i32 addrspace(5)* %tmp1 to [4 x i32] addrspace(5)*
+  %tmp3 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp2, i32 0, i32 0
+  %tmp4 = load i32, i32 addrspace(5)* %tmp3
   store i32 %tmp4, i32 addrspace(1)* %out
   ret void
 }
@@ -95,18 +96,18 @@ entry:
 ; OPT: store i32 %0, i32 addrspace(1)* %out, align 4
 define amdgpu_kernel void @vector_read_bitcast_gep(i32 addrspace(1)* %out, i32 %index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %x = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 0
-  %y = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %z = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 2
-  %w = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 3
-  %bc = bitcast i32* %x to float*
-  store float 1.0, float* %bc
-  store i32 1, i32* %y
-  store i32 2, i32* %z
-  store i32 3, i32* %w
-  %tmp1 = getelementptr inbounds [4 x i32], [4 x i32]* %tmp, i32 0, i32 %index
-  %tmp2 = load i32, i32* %tmp1
+  %tmp = alloca [4 x i32], addrspace(5)
+  %x = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %y = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %z = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2
+  %w = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 3
+  %bc = bitcast i32 addrspace(5)* %x to float addrspace(5)*
+  store float 1.0, float addrspace(5)* %bc
+  store i32 1, i32 addrspace(5)* %y
+  store i32 2, i32 addrspace(5)* %z
+  store i32 3, i32 addrspace(5)* %w
+  %tmp1 = getelementptr inbounds [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i32, i32 addrspace(5)* %tmp1
   store i32 %tmp2, i32 addrspace(1)* %out
   ret void
 }
@@ -123,18 +124,18 @@ entry:
 ; OPT: load float
 define amdgpu_kernel void @vector_read_bitcast_alloca(float addrspace(1)* %out, i32 %index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %tmp.bc = bitcast [4 x i32]* %tmp to [4 x float]*
-  %x = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 0
-  %y = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 1
-  %z = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 2
-  %w = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 3
-  store float 0.0, float* %x
-  store float 1.0, float* %y
-  store float 2.0, float* %z
-  store float 4.0, float* %w
-  %tmp1 = getelementptr inbounds [4 x float], [4 x float]* %tmp.bc, i32 0, i32 %index
-  %tmp2 = load float, float* %tmp1
+  %tmp = alloca [4 x i32], addrspace(5)
+  %tmp.bc = bitcast [4 x i32] addrspace(5)* %tmp to [4 x float] addrspace(5)*
+  %x = getelementptr inbounds [4 x float], [4 x float] addrspace(5)* %tmp.bc, i32 0, i32 0
+  %y = getelementptr inbounds [4 x float], [4 x float] addrspace(5)* %tmp.bc, i32 0, i32 1
+  %z = getelementptr inbounds [4 x float], [4 x float] addrspace(5)* %tmp.bc, i32 0, i32 2
+  %w = getelementptr inbounds [4 x float], [4 x float] addrspace(5)* %tmp.bc, i32 0, i32 3
+  store float 0.0, float addrspace(5)* %x
+  store float 1.0, float addrspace(5)* %y
+  store float 2.0, float addrspace(5)* %z
+  store float 4.0, float addrspace(5)* %w
+  %tmp1 = getelementptr inbounds [4 x float], [4 x float] addrspace(5)* %tmp.bc, i32 0, i32 %index
+  %tmp2 = load float, float addrspace(5)* %tmp1
   store float %tmp2, float addrspace(1)* %out
   ret void
 }
@@ -146,17 +147,17 @@ entry:
 ; OPT: store i32 %0, i32 addrspace(1)* %out, align 4
 define amdgpu_kernel void @vector_read_with_local_arg(i32 addrspace(3)* %stopper, i32 addrspace(1)* %out, i32 %index) {
 entry:
-  %tmp = alloca [4 x i32]
-  %x = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 0
-  %y = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 1
-  %z = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 2
-  %w = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 3
-  store i32 0, i32* %x
-  store i32 1, i32* %y
-  store i32 2, i32* %z
-  store i32 3, i32* %w
-  %tmp1 = getelementptr [4 x i32], [4 x i32]* %tmp, i32 0, i32 %index
-  %tmp2 = load i32, i32* %tmp1
+  %tmp = alloca [4 x i32], addrspace(5)
+  %x = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 0
+  %y = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 1
+  %z = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 2
+  %w = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 3
+  store i32 0, i32 addrspace(5)* %x
+  store i32 1, i32 addrspace(5)* %y
+  store i32 2, i32 addrspace(5)* %z
+  store i32 3, i32 addrspace(5)* %w
+  %tmp1 = getelementptr [4 x i32], [4 x i32] addrspace(5)* %tmp, i32 0, i32 %index
+  %tmp2 = load i32, i32 addrspace(5)* %tmp1
   store i32 %tmp2, i32 addrspace(1)* %out
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/vector-extract-insert.ll b/test/CodeGen/AMDGPU/vector-extract-insert.ll
index ab2bfcfd1fb71..e3027d0930998 100644
--- a/test/CodeGen/AMDGPU/vector-extract-insert.ll
+++ b/test/CodeGen/AMDGPU/vector-extract-insert.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; Test that when extracting the same unknown vector index from an
 ; insertelement the dynamic indexing is folded away.
@@ -81,4 +81,4 @@ define amdgpu_kernel void @extract_insert_same_dynelt_v4f32(float addrspace(1)*
 }
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind }
-\ No newline at end of file
+attributes #1 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
index feae5e9f37920..a0242ec958b3f 100644
--- a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
+++ b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=SIMESA %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+vgpr-spilling,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=VIMESA %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+vgpr-spilling,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=GFX9MESA %s
-; RUN: llc -march=amdgcn -mcpu=hawaii -mtriple=amdgcn-unknown-amdhsa -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CIHSA -check-prefix=HSA %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mtriple=amdgcn-unknown-amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VIHSA -check-prefix=HSA %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tahiti -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=SIMESA %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=fiji -mattr=+vgpr-spilling,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=VIMESA %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=+vgpr-spilling,-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCNMESA -check-prefix=GFX9MESA %s
+; RUN: llc -march=amdgcn  -mcpu=hawaii -mtriple=amdgcn-unknown-amdhsa-amdgiz -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CIHSA -check-prefix=HSA %s
+; RUN: llc -march=amdgcn  -mcpu=fiji -mtriple=amdgcn-unknown-amdhsa-amdgiz -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VIHSA -check-prefix=HSA %s
 
 ; This ends up using all 256 registers and requires register
 ; scavenging which will fail to find an unsued register.
diff --git a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
index afbd06a00faed..89327fb8f80de 100644
--- a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
+++ b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=amdgcn -mcpu=tahiti -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tahiti -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=fiji -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 %s
 
 ; This ends up using all 255 registers and requires register
 ; scavenging which will fail to find an unsued register.
diff --git a/test/CodeGen/AMDGPU/vop-shrink-frame-index.mir b/test/CodeGen/AMDGPU/vop-shrink-frame-index.mir
index f8a2339626cf1..1c34789ed6006 100644
--- a/test/CodeGen/AMDGPU/vop-shrink-frame-index.mir
+++ b/test/CodeGen/AMDGPU/vop-shrink-frame-index.mir
@@ -33,9 +33,9 @@
 
 ...
 # GCN-LABEL: name: fold_fi_vgpr{{$}}
-# GCN: %1 = IMPLICIT_DEF
+# GCN: %1:vgpr_32 = IMPLICIT_DEF
 
-# GCN: %2 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
 name: fold_fi_vgpr
 tracksRegLiveness: true
 registers:
@@ -55,8 +55,8 @@ body:             |
 
 ...
 # GCN-LABEL: name: fold_vgpr_fi{{$}}
-# GCN: %1 = IMPLICIT_DEF
-# GCN: %2 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
+# GCN: %1:vgpr_32 = IMPLICIT_DEF
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
 name: fold_vgpr_fi
 tracksRegLiveness: true
 registers:
@@ -76,9 +76,9 @@ body:             |
 
 ...
 # GCN-LABEL: name: fold_sgpr_fi{{$}}
-# GCN: %0 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
-# GCN: %1 = IMPLICIT_DEF
-# GCN: %2 = V_ADD_I32_e32 %1, %0, implicit-def %vcc, implicit %exec
+# GCN: %0:vgpr_32 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
+# GCN: %1:sgpr_32 = IMPLICIT_DEF
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 %1, %0, implicit-def %vcc, implicit %exec
 name: fold_sgpr_fi
 tracksRegLiveness: true
 registers:
@@ -98,9 +98,9 @@ body:             |
 
 ...
 # GCN-LABEL: name: fold_fi_sgpr{{$}}
-# GCN: %0 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
-# GCN: %1 = IMPLICIT_DEF
-# GCN: %2 = V_ADD_I32_e32 %1, %0, implicit-def %vcc, implicit %exec
+# GCN: %0:vgpr_32 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
+# GCN: %1:sgpr_32 = IMPLICIT_DEF
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 %1, %0, implicit-def %vcc, implicit %exec
 name: fold_fi_sgpr
 tracksRegLiveness: true
 registers:
@@ -120,8 +120,8 @@ body:             |
 ...
 # TODO: Should probably prefer folding immediate first
 # GCN-LABEL: name: fold_fi_imm{{$}}
-# GCN: %1 = V_MOV_B32_e32 999, implicit %exec
-# GCN: %2 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
+# GCN: %1:vgpr_32 = V_MOV_B32_e32 999, implicit %exec
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 %stack.0.alloca, %1, implicit-def %vcc, implicit %exec
 name: fold_fi_imm
 tracksRegLiveness: true
 registers:
@@ -141,8 +141,8 @@ body:             |
 
 ...
 # GCN-LABEL: name: fold_imm_fi{{$}}
-# GCN: %0 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
-# GCN: %2 = V_ADD_I32_e32 999, %0, implicit-def %vcc, implicit %exec
+# GCN: %0:vgpr_32 = V_MOV_B32_e32 %stack.0.alloca, implicit %exec
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 999, %0, implicit-def %vcc, implicit %exec
 name: fold_imm_fi
 tracksRegLiveness: true
 registers:
diff --git a/test/CodeGen/AMDGPU/vop-shrink-non-ssa.mir b/test/CodeGen/AMDGPU/vop-shrink-non-ssa.mir
index b4c0c93347c20..a190324cdc25f 100644
--- a/test/CodeGen/AMDGPU/vop-shrink-non-ssa.mir
+++ b/test/CodeGen/AMDGPU/vop-shrink-non-ssa.mir
@@ -1,8 +1,8 @@
 # RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-shrink-instructions -o - %s | FileCheck -check-prefix=GCN %s
 ...
 # GCN-LABEL: name: fold_imm_non_ssa{{$}}
-# GCN: %0 = V_MOV_B32_e32 123, implicit %exec
-# GCN: %2 = V_ADD_I32_e32 456, %0, implicit-def %vcc, implicit %exec
+# GCN: %0:vgpr_32 = V_MOV_B32_e32 123, implicit %exec
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 456, %0, implicit-def %vcc, implicit %exec
 
 name: fold_imm_non_ssa
 tracksRegLiveness: true
@@ -21,8 +21,8 @@ body:             |
 
 ...
 # GCN-LABEL: name: fold_partially_defined_superreg{{$}}
-# GCN: %1 = V_MOV_B32_e32 456, implicit %exec
-# GCN: %2 = V_ADD_I32_e32 123, %1, implicit-def %vcc, implicit %exec
+# GCN: %1:vgpr_32 = V_MOV_B32_e32 456, implicit %exec
+# GCN: %2:vgpr_32 = V_ADD_I32_e32 123, %1, implicit-def %vcc, implicit %exec
 name: fold_partially_defined_superreg
 tracksRegLiveness: true
 registers:
diff --git a/test/CodeGen/AMDGPU/vop-shrink.ll b/test/CodeGen/AMDGPU/vop-shrink.ll
index d2708b068eb42..1c2776ed4fca2 100644
--- a/test/CodeGen/AMDGPU/vop-shrink.ll
+++ b/test/CodeGen/AMDGPU/vop-shrink.ll
@@ -4,7 +4,7 @@
 ; Test that we correctly commute a sub instruction
 ; FUNC-LABEL: {{^}}sub_rev:
 ; SI-NOT: v_sub_i32_e32 v{{[0-9]+}}, vcc, s
-; SI: v_subrev_i32_e32 v{{[0-9]+}}, vcc, s
+; SI: v_subrev_{{[iu]}}32_e32 v{{[0-9]+}}, vcc, s
 
 ; ModuleID = 'vop-shrink.ll'
 
diff --git a/test/CodeGen/AMDGPU/waitcnt-flat.ll b/test/CodeGen/AMDGPU/waitcnt-flat.ll
index 5d86b12da95fd..813255a24a1e6 100644
--- a/test/CodeGen/AMDGPU/waitcnt-flat.ll
+++ b/test/CodeGen/AMDGPU/waitcnt-flat.ll
@@ -1,5 +1,6 @@
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=GCN %s
-; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global | FileCheck --check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; If flat_store_dword and flat_load_dword use different registers for the data
 ; operand, this test is not broken.  It just means it is no longer testing
@@ -9,8 +10,19 @@
 ; XGCN: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[DATA:v[0-9]+]]
 ; XGCN: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; XGCN: flat_load_dword [[DATA]], v[{{[0-9]+:[0-9]+}}]
-define amdgpu_kernel void @test(i32 addrspace(1)* %out, i32 %in) {
-  store volatile i32 0, i32 addrspace(1)* %out
-  %val = load volatile i32, i32 addrspace(1)* %out
+define amdgpu_kernel void @test(i32 addrspace(4)* %out, i32 %in) {
+  store volatile i32 0, i32 addrspace(4)* %out
+  %val = load volatile i32, i32 addrspace(4)* %out
+  ret void
+}
+
+; Make sure lgkmcnt isn't used for global_* instructions
+; GCN-LABEL: {{^}}test_waitcnt_type_flat_global:
+; GFX9: global_load_dword [[LD:v[0-9]+]]
+; GFX9-NEXT: s_waitcnt vmcnt(0){{$}}
+; GFX9-NEXT: ds_write_b32 [[LD]]
+define amdgpu_kernel void @test_waitcnt_type_flat_global(i32 addrspace(1)* %in) {
+  %val = load volatile i32, i32 addrspace(1)* %in
+  store volatile i32 %val, i32 addrspace(3)* undef
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll b/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
new file mode 100644
index 0000000000000..4ae219949c220
--- /dev/null
+++ b/test/CodeGen/AMDGPU/widen-vselect-and-mask.ll
@@ -0,0 +1,52 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+; Check that DAGTypeLegalizer::WidenVSELECTAndMask doesn't try to
+; create vselects with i64 condition masks.
+
+; FIXME: Should be able to avoid intermediate vselect
+; GCN-LABEL: {{^}}widen_vselect_and_mask_v4f64:
+; GCN: v_cmp_u_f64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
+; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
+; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
+; GCN: v_cmp_lt_i64_e32 vcc, -1, v{{\[}}[[VSEL]]:[[VSEL_EXT]]{{\]}}
+define amdgpu_kernel void @widen_vselect_and_mask_v4f64(<4 x double> %arg) #0 {
+bb:
+  %tmp = extractelement <4 x double> %arg, i64 0
+  %tmp1 = fcmp uno double %tmp, 0.000000e+00
+  %tmp2 = sext i1 %tmp1 to i64
+  %tmp3 = insertelement <4 x i64> undef, i64 %tmp2, i32 0
+  %tmp4 = insertelement <4 x i64> %tmp3, i64 undef, i32 1
+  %tmp5 = insertelement <4 x i64> %tmp4, i64 undef, i32 2
+  %tmp6 = insertelement <4 x i64> %tmp5, i64 undef, i32 3
+  %tmp7 = fcmp une <4 x double> %arg, zeroinitializer
+  %tmp8 = icmp sgt <4 x i64> %tmp6, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %tmp9 = and <4 x i1> %tmp8, %tmp7
+  %tmp10 = select <4 x i1> %tmp9, <4 x double> <double 1.0, double 1.0, double 1.0, double 1.0>, <4 x double> zeroinitializer
+  store <4 x double> %tmp10, <4 x double> addrspace(1)* null, align 32
+  ret void
+}
+
+; GCN-LABEL: {{^}}widen_vselect_and_mask_v4i64:
+; GCN: v_cmp_eq_u64_e64 [[CMP:s\[[0-9]+:[0-9]+\]]],
+; GCN: v_cndmask_b32_e64 v[[VSEL:[0-9]+]], 0, -1, [[CMP]]
+; GCN: v_mov_b32_e32 v[[VSEL_EXT:[0-9]+]], v[[VSEL]]
+; GCN: v_cmp_lt_i64_e32 vcc, -1, v{{\[}}[[VSEL]]:[[VSEL_EXT]]{{\]}}
+define amdgpu_kernel void @widen_vselect_and_mask_v4i64(<4 x i64> %arg) #0 {
+bb:
+  %tmp = extractelement <4 x i64> %arg, i64 0
+  %tmp1 = icmp eq i64 %tmp, 0
+  %tmp2 = sext i1 %tmp1 to i64
+  %tmp3 = insertelement <4 x i64> undef, i64 %tmp2, i32 0
+  %tmp4 = insertelement <4 x i64> %tmp3, i64 undef, i32 1
+  %tmp5 = insertelement <4 x i64> %tmp4, i64 undef, i32 2
+  %tmp6 = insertelement <4 x i64> %tmp5, i64 undef, i32 3
+  %tmp7 = icmp ne <4 x i64> %arg, zeroinitializer
+  %tmp8 = icmp sgt <4 x i64> %tmp6, <i64 -1, i64 -1, i64 -1, i64 -1>
+  %tmp9 = and <4 x i1> %tmp8, %tmp7
+  %tmp10 = select <4 x i1> %tmp9, <4 x i64> <i64 1, i64 1, i64 1, i64 1>, <4 x i64> zeroinitializer
+  store <4 x i64> %tmp10, <4 x i64> addrspace(1)* null, align 32
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone speculatable }
diff --git a/test/CodeGen/AMDGPU/widen_extending_scalar_loads.ll b/test/CodeGen/AMDGPU/widen_extending_scalar_loads.ll
new file mode 100644
index 0000000000000..e200f7b64d19c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/widen_extending_scalar_loads.ll
@@ -0,0 +1,192 @@
+; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare < %s | FileCheck -check-prefix=OPT %s
+
+declare i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #0
+
+; OPT-LABEL: @constant_load_i1
+; OPT: load i1
+; OPT-NEXT: store i1
+define amdgpu_kernel void @constant_load_i1(i1 addrspace(1)* %out, i1 addrspace(2)* %in) #0 {
+  %val = load i1, i1 addrspace(2)* %in
+  store i1 %val, i1 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i1_align2
+; OPT: load i1
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i1_align2(i1 addrspace(1)* %out, i1 addrspace(2)* %in) #0 {
+  %val = load i1, i1 addrspace(2)* %in, align 2
+  store i1 %val, i1 addrspace(1)* %out, align 2
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i1_align4
+; OPT: bitcast
+; OPT-NEXT: load i32
+; OPT-NEXT: trunc
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i1_align4(i1 addrspace(1)* %out, i1 addrspace(2)* %in) #0 {
+  %val = load i1, i1 addrspace(2)* %in, align 4
+  store i1 %val, i1 addrspace(1)* %out, align 4
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i8
+; OPT: load i8
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i8(i8 addrspace(1)* %out, i8 addrspace(2)* %in) #0 {
+  %val = load i8, i8 addrspace(2)* %in
+  store i8 %val, i8 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i8_align2
+; OPT: load i8
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i8_align2(i8 addrspace(1)* %out, i8 addrspace(2)* %in) #0 {
+  %val = load i8, i8 addrspace(2)* %in, align 2
+  store i8 %val, i8 addrspace(1)* %out, align 2
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i8align4
+; OPT: bitcast
+; OPT-NEXT: load i32
+; OPT-NEXT: trunc
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i8align4(i8 addrspace(1)* %out, i8 addrspace(2)* %in) #0 {
+  %val = load i8, i8 addrspace(2)* %in, align 4
+  store i8 %val, i8 addrspace(1)* %out, align 4
+  ret void
+}
+
+
+; OPT-LABEL: @constant_load_v2i8
+; OPT: load <2 x i8>
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 {
+  %ld = load <2 x i8>, <2 x i8> addrspace(2)* %in
+  store <2 x i8> %ld, <2 x i8> addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v2i8_align4
+; OPT: bitcast
+; OPT-NEXT: load i32
+; OPT-NEXT: trunc
+; OPT-NEXT: bitcast
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_v2i8_align4(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 {
+  %ld = load <2 x i8>, <2 x i8> addrspace(2)* %in, align 4
+  store <2 x i8> %ld, <2 x i8> addrspace(1)* %out, align 4
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v3i8
+; OPT: bitcast <3 x i8>
+; OPT-NEXT: load i32, i32 addrspace(2)
+; OPT-NEXT: trunc i32
+; OPT-NEXT: bitcast i24
+; OPT-NEXT: store <3 x i8>
+define amdgpu_kernel void @constant_load_v3i8(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 {
+  %ld = load <3 x i8>, <3 x i8> addrspace(2)* %in
+  store <3 x i8> %ld, <3 x i8> addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v3i8_align4
+; OPT: bitcast <3 x i8>
+; OPT-NEXT: load i32, i32 addrspace(2)
+; OPT-NEXT: trunc i32
+; OPT-NEXT: bitcast i24
+; OPT-NEXT: store <3 x i8>
+define amdgpu_kernel void @constant_load_v3i8_align4(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 {
+  %ld = load <3 x i8>, <3 x i8> addrspace(2)* %in, align 4
+  store <3 x i8> %ld, <3 x i8> addrspace(1)* %out, align 4
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i16
+; OPT: load i16
+; OPT: sext
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i16(i32 addrspace(1)* %out, i16 addrspace(2)* %in) #0 {
+  %ld = load i16, i16 addrspace(2)* %in
+  %ext = sext i16 %ld to i32
+  store i32 %ext, i32 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_i16_align4
+; OPT: bitcast
+; OPT-NEXT: load i32
+; OPT-NEXT: trunc
+; OPT-NEXT: sext
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_i16_align4(i32 addrspace(1)* %out, i16 addrspace(2)* %in) #0 {
+  %ld = load i16, i16 addrspace(2)* %in, align 4
+  %ext = sext i16 %ld to i32
+  store i32 %ext, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; OPT-LABEL: @constant_load_f16
+; OPT: load half
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_f16(half addrspace(1)* %out, half addrspace(2)* %in) #0 {
+  %ld = load half, half addrspace(2)* %in
+  store half %ld, half addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v2f16
+; OPT: load <2 x half>
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_v2f16(<2 x half> addrspace(1)* %out, <2 x half> addrspace(2)* %in) #0 {
+  %ld = load <2 x half>, <2 x half> addrspace(2)* %in
+  store <2 x half> %ld, <2 x half> addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @load_volatile
+; OPT: load volatile i16
+; OPT-NEXT: store
+define amdgpu_kernel void @load_volatile(i16 addrspace(1)* %out, i16 addrspace(2)* %in) {
+  %a = load volatile i16, i16 addrspace(2)* %in
+  store i16 %a, i16 addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v2i8_volatile
+; OPT: load volatile <2 x i8>
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_v2i8_volatile(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 {
+  %ld = load volatile <2 x i8>, <2 x i8> addrspace(2)* %in
+  store <2 x i8> %ld, <2 x i8> addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @constant_load_v2i8_addrspace1
+; OPT: load <2 x i8>
+; OPT-NEXT: store
+define amdgpu_kernel void @constant_load_v2i8_addrspace1(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 {
+  %ld = load <2 x i8>, <2 x i8> addrspace(1)* %in
+  store <2 x i8> %ld, <2 x i8> addrspace(1)* %out
+  ret void
+}
+
+; OPT-LABEL: @use_dispatch_ptr
+; OPT: bitcast
+; OPT-NEXT: load i32
+; OPT-NEXT: trunc
+; OPT-NEXT: zext
+; OPT-NEXT: store
+define amdgpu_kernel void @use_dispatch_ptr(i32 addrspace(1)* %ptr) #1 {
+  %dispatch.ptr = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr()
+  %val = load i8, i8 addrspace(2)* %dispatch.ptr, align 4
+  %ld = zext i8 %val to i32
+  store i32 %ld, i32 addrspace(1)* %ptr
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/wqm.ll b/test/CodeGen/AMDGPU/wqm.ll
index 133aaa35981e1..166901fe6c209 100644
--- a/test/CodeGen/AMDGPU/wqm.ll
+++ b/test/CodeGen/AMDGPU/wqm.ll
@@ -12,24 +12,33 @@ main_body:
   ret <4 x float> %tex
 }
 
-; Check that WQM is triggered by image samples and left untouched for loads...
+; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible
 ;
 ;CHECK-LABEL: {{^}}test2:
 ;CHECK-NEXT: ; %main_body
+;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
 ;CHECK-NEXT: s_wqm_b64 exec, exec
+;CHECK: interp
+;CHECK: s_and_b64 exec, exec, [[ORIG]]
+;CHECK-NOT: interp
+;CHECK: image_sample
 ;CHECK-NOT: exec
-define amdgpu_ps void @test2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <4 x float> %c) {
+;CHECK: .size test2
+define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {
 main_body:
-  %c.1 = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0
-  %c.2 = bitcast <4 x float> %c.1 to <4 x i32>
-  %c.3 = extractelement <4 x i32> %c.2, i32 0
-  %gep = getelementptr float, float addrspace(1)* %ptr, i32 %c.3
-  %data = load float, float addrspace(1)* %gep
-  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %data, float undef, float undef, float undef, i1 true, i1 true) #1
-  ret void
+  %inst23 = extractelement <2 x float> %pos, i32 0
+  %inst24 = extractelement <2 x float> %pos, i32 1
+  %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)
+  %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)
+  %inst27 = insertelement <2 x float> undef, float %inst26, i32 0
+  %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)
+  %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)
+  %inst30 = insertelement <2 x float> %inst27, float %inst29, i32 1
+  %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> %inst30, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0
+  ret <4 x float> %tex
 }
 
-; ... but disabled for stores (and, in this simple case, not re-enabled).
+; ... but disabled for stores (and, in this simple case, not re-enabled) ...
 ;
 ;CHECK-LABEL: {{^}}test3:
 ;CHECK-NEXT: ; %main_body
@@ -51,6 +60,36 @@ main_body:
   ret <4 x float> %tex
 }
 
+; ... and disabled for export.
+;
+;CHECK-LABEL: {{^}}test3x:
+;CHECK-NEXT: ; %main_body
+;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK-NEXT: s_wqm_b64 exec, exec
+;CHECK: s_and_b64 exec, exec, [[ORIG]]
+;CHECK: image_sample
+;CHECK: exp
+;CHECK-NOT: exec
+;CHECK: .size test3x
+define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {
+main_body:
+  %inst23 = extractelement <2 x float> %pos, i32 0
+  %inst24 = extractelement <2 x float> %pos, i32 1
+  %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)
+  %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)
+  %inst27 = insertelement <2 x float> undef, float %inst26, i32 0
+  %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)
+  %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)
+  %inst30 = insertelement <2 x float> %inst27, float %inst29, i32 1
+  %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float> %inst30, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0
+  %tex.0 = extractelement <4 x float> %tex, i32 0
+  %tex.1 = extractelement <4 x float> %tex, i32 1
+  %tex.2 = extractelement <4 x float> %tex, i32 2
+  %tex.3 = extractelement <4 x float> %tex, i32 3
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true)
+  ret void
+}
+
 ; Check that WQM is re-enabled when required.
 ;
 ;CHECK-LABEL: {{^}}test4:
@@ -74,6 +113,229 @@ main_body:
   ret <4 x float> %dtex
 }
 
+; Check that WQM is triggered by the wqm intrinsic.
+;
+;CHECK-LABEL: {{^}}test5:
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
+  ret float %out.0
+}
+
+; Check that the wqm intrinsic works correctly for integers.
+;
+;CHECK-LABEL: {{^}}test6:
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %out.0 = bitcast float %out to i32
+  %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0)
+  %out.2 = bitcast i32 %out.1 to float
+  ret float %out.2
+}
+
+; Check that WWM is triggered by the wwm intrinsic.
+;
+;CHECK-LABEL: {{^}}test_wwm1:
+;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
+  ret float %out.0
+}
+
+; Same as above, but with an integer type.
+;
+;CHECK-LABEL: {{^}}test_wwm2:
+;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_{{[iu]}}32_e32
+define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %src0.0 = bitcast float %src0 to i32
+  %src1.0 = bitcast float %src1 to i32
+  %out = add i32 %src0.0, %src1.0
+  %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)
+  %out.1 = bitcast i32 %out.0 to float
+  ret float %out.1
+}
+
+; Check that we don't leave WWM on for computations that don't require WWM,
+; since that will lead clobbering things that aren't supposed to be clobbered
+; in cases like this.
+;
+;CHECK-LABEL: {{^}}test_wwm3:
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_wwm3(i32 inreg %idx) {
+main_body:
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
+  %out = fadd float %src, %src
+  %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
+  %out.1 = fadd float %src, %out.0
+  br label %endif
+
+endif:
+  %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]
+  ret float %out.2
+}
+
+; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM
+; write could clobber disabled channels in the non-WWM one.
+;
+;CHECK-LABEL: {{^}}test_wwm4:
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK-NEXT: v_mov_b32_e32
+define amdgpu_ps float @test_wwm4(i32 inreg %idx) {
+main_body:
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
+  %out = fadd float %src, %src
+  %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
+  br label %endif
+
+endif:
+  %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+  ret float %out.1
+}
+
+; Make sure the transition from Exact to WWM then WQM works properly.
+;
+;CHECK-LABEL: {{^}}test_wwm5:
+;CHECK: buffer_load_dword
+;CHECK: buffer_store_dword
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: s_wqm_b64 exec, exec
+define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  call void @llvm.amdgcn.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %temp = fadd float %src1, %src1
+  %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp)
+  %out = fadd float %temp.0, %temp.0
+  %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
+  ret float %out.0
+}
+
+; Check that WWM is turned on correctly across basic block boundaries.
+;
+;CHECK-LABEL: {{^}}test_wwm6:
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: %if
+;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+define amdgpu_ps float @test_wwm6() {
+main_body:
+  %src0 = load volatile float, float addrspace(1)* undef
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src1 = load volatile float, float addrspace(1)* undef
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
+  br label %endif
+
+endif:
+  %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+  ret float %out.1
+}
+
+; Check that @llvm.amdgcn.set.inactive disables WWM.
+;
+;CHECK-LABEL: {{^}}test_set_inactive1:
+;CHECK: buffer_load_dword
+;CHECK: s_not_b64 exec, exec
+;CHECK: v_mov_b32_e32
+;CHECK: s_not_b64 exec, exec
+;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
+;CHECK: v_add_{{[iu]}}32_e32
+define amdgpu_ps void @test_set_inactive1(i32 inreg %idx) {
+main_body:
+  %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
+  %src.0 = bitcast float %src to i32
+  %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0)
+  %out = add i32 %src.1, %src.1
+  %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)
+  %out.1 = bitcast i32 %out.0 to float
+  call void @llvm.amdgcn.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
+  ret void
+}
+
+; Check that enabling WQM anywhere enables WQM for the set.inactive source.
+;
+;CHECK-LABEL: {{^}}test_set_inactive2:
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  %src1.0 = bitcast float %src1 to i32
+  %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef)
+  %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
+  %src0.0 = bitcast float %src0 to i32
+  %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0)
+  %out = add i32 %src0.1, %src1.1
+  %out.0 = bitcast i32 %out to float
+  call void @llvm.amdgcn.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
+  ret void
+}
+
 ; Check a case of one branch of an if-else requiring WQM, the other requiring
 ; exact.
 ;
@@ -211,8 +473,8 @@ END:
 ;CHECK: image_sample
 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
 ;CHECK: image_sample
-;CHECK: v_cmp
-;CHECK: store
+;CHECK-DAG: v_cmp
+;CHECK-DAG: store
 define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) {
 main_body:
   %tex = call <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i1 false, i1 false, i1 false, i1 false, i1 false) #0
@@ -331,7 +593,7 @@ main_body:
 ; CHECK: s_wqm_b64 exec, exec
 ; CHECK: v_add_f32_e32 v0,
 ; CHECK: s_and_b64 exec, exec, [[ORIG]]
-define amdgpu_ps float @test_prolog_1(float %a, float %b) #4 {
+define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 {
 main_body:
   %s = fadd float %a, %b
   ret float %s
@@ -494,8 +756,21 @@ declare <4 x float> @llvm.amdgcn.image.sample.v4f32.f32.v8i32(float, <8 x i32>,
 declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #3
 declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #3
 declare void @llvm.AMDGPU.kill(float) #1
+declare float @llvm.amdgcn.wqm.f32(float) #3
+declare i32 @llvm.amdgcn.wqm.i32(i32) #3
+declare float @llvm.amdgcn.wwm.f32(float) #3
+declare i32 @llvm.amdgcn.wwm.i32(i32) #3
+declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3
+declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3
+declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1
+declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2
+declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2
 
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readonly }
 attributes #3 = { nounwind readnone }
-attributes #4 = { "amdgpu-ps-wqm-outputs" }
+attributes #4 = { nounwind readnone convergent }
+attributes #5 = { "amdgpu-ps-wqm-outputs" }
+attributes #6 = { nounwind "InitialPSInputAddr"="2" }
diff --git a/test/CodeGen/AMDGPU/wqm.mir b/test/CodeGen/AMDGPU/wqm.mir
new file mode 100644
index 0000000000000..637a00346bd0a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/wqm.mir
@@ -0,0 +1,50 @@
+# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-wqm -o -  %s | FileCheck %s
+
+---
+# Check for awareness that s_or_saveexec_b64 clobbers SCC
+#
+#CHECK: S_OR_SAVEEXEC_B64
+#CHECK: S_CMP_LT_I32
+#CHECK: S_CSELECT_B32
+name:            test_wwm_scc
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:       
+  - { id: 0, class: sgpr_32, preferred-register: '' }
+  - { id: 1, class: sgpr_32, preferred-register: '' }
+  - { id: 2, class: sgpr_32, preferred-register: '' }
+  - { id: 3, class: vgpr_32, preferred-register: '' }
+  - { id: 4, class: vgpr_32, preferred-register: '' }
+  - { id: 5, class: sgpr_32, preferred-register: '' }
+  - { id: 6, class: vgpr_32, preferred-register: '' }
+  - { id: 7, class: vgpr_32, preferred-register: '' }
+  - { id: 8, class: sreg_32_xm0, preferred-register: '' }
+  - { id: 9, class: sreg_32, preferred-register: '' }
+  - { id: 10, class: sreg_32, preferred-register: '' }
+  - { id: 11, class: vgpr_32, preferred-register: '' }
+  - { id: 12, class: vgpr_32, preferred-register: '' }
+liveins:         
+  - { reg: '%sgpr0', virtual-reg: '%0' }
+  - { reg: '%sgpr1', virtual-reg: '%1' }
+  - { reg: '%sgpr2', virtual-reg: '%2' }
+  - { reg: '%vgpr0', virtual-reg: '%3' }
+body:             |
+  bb.0:
+    liveins: %sgpr0, %sgpr1, %sgpr2, %vgpr0
+  
+    %3 = COPY %vgpr0
+    %2 = COPY %sgpr2
+    %1 = COPY %sgpr1
+    %0 = COPY %sgpr0
+    S_CMP_LT_I32 0, %0, implicit-def %scc
+    %12 = V_ADD_I32_e32 %3, %3, implicit-def %vcc, implicit %exec
+    %5 = S_CSELECT_B32 %2, %1, implicit %scc
+    %11 = V_ADD_I32_e32 %5, %12, implicit-def %vcc, implicit %exec
+    %vgpr0 = WWM %11, implicit %exec
+    SI_RETURN_TO_EPILOG %vgpr0
+
+...
diff --git a/test/CodeGen/AMDGPU/write-register-vgpr-into-sgpr.ll b/test/CodeGen/AMDGPU/write-register-vgpr-into-sgpr.ll
index b1ee016e99c9c..51d9efee26815 100644
--- a/test/CodeGen/AMDGPU/write-register-vgpr-into-sgpr.ll
+++ b/test/CodeGen/AMDGPU/write-register-vgpr-into-sgpr.ll
@@ -8,15 +8,17 @@
 
 declare void @llvm.write_register.i32(metadata, i32) #0
 declare i32 @llvm.amdgcn.workitem.id.x() #0
-
+declare void @llvm.amdgcn.wave.barrier() #2
 
 define amdgpu_kernel void @write_vgpr_into_sgpr() {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   call void @llvm.write_register.i32(metadata !0, i32 %tid)
+  call void @llvm.amdgcn.wave.barrier() #2
   ret void
 }
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
+attributes #2 = { convergent nounwind }
 
 !0 = !{!"exec_lo"}
diff --git a/test/CodeGen/AMDGPU/write_register.ll b/test/CodeGen/AMDGPU/write_register.ll
index 9c62e003dde0f..0ca92f90c225a 100644
--- a/test/CodeGen/AMDGPU/write_register.ll
+++ b/test/CodeGen/AMDGPU/write_register.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -enable-misched=0 -verify-machineinstrs < %s | FileCheck %s
 
 declare void @llvm.write_register.i32(metadata, i32) #0
 declare void @llvm.write_register.i64(metadata, i64) #0
@@ -8,6 +8,7 @@ define amdgpu_kernel void @test_write_m0(i32 %val) #0 {
   call void @llvm.write_register.i32(metadata !0, i32 0)
   call void @llvm.write_register.i32(metadata !0, i32 -1)
   call void @llvm.write_register.i32(metadata !0, i32 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -19,6 +20,7 @@ define amdgpu_kernel void @test_write_exec(i64 %val) #0 {
   call void @llvm.write_register.i64(metadata !1, i64 0)
   call void @llvm.write_register.i64(metadata !1, i64 -1)
   call void @llvm.write_register.i64(metadata !1, i64 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -30,6 +32,7 @@ define amdgpu_kernel void @test_write_flat_scratch(i64 %val) #0 {
   call void @llvm.write_register.i64(metadata !2, i64 0)
   call void @llvm.write_register.i64(metadata !2, i64 -1)
   call void @llvm.write_register.i64(metadata !2, i64 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -39,6 +42,7 @@ define amdgpu_kernel void @test_write_flat_scratch(i64 %val) #0 {
 define amdgpu_kernel void @test_write_flat_scratch_lo(i32 %val) #0 {
   call void @llvm.write_register.i32(metadata !3, i32 0)
   call void @llvm.write_register.i32(metadata !3, i32 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -48,6 +52,7 @@ define amdgpu_kernel void @test_write_flat_scratch_lo(i32 %val) #0 {
 define amdgpu_kernel void @test_write_flat_scratch_hi(i32 %val) #0 {
   call void @llvm.write_register.i32(metadata !4, i32 0)
   call void @llvm.write_register.i32(metadata !4, i32 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -57,6 +62,7 @@ define amdgpu_kernel void @test_write_flat_scratch_hi(i32 %val) #0 {
 define amdgpu_kernel void @test_write_exec_lo(i32 %val) #0 {
   call void @llvm.write_register.i32(metadata !5, i32 0)
   call void @llvm.write_register.i32(metadata !5, i32 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
@@ -66,10 +72,14 @@ define amdgpu_kernel void @test_write_exec_lo(i32 %val) #0 {
 define amdgpu_kernel void @test_write_exec_hi(i32 %val) #0 {
   call void @llvm.write_register.i32(metadata !6, i32 0)
   call void @llvm.write_register.i32(metadata !6, i32 %val)
+  call void @llvm.amdgcn.wave.barrier() #1
   ret void
 }
 
+declare void @llvm.amdgcn.wave.barrier() #1
+
 attributes #0 = { nounwind }
+attributes #1 = { convergent nounwind }
 
 !0 = !{!"m0"}
 !1 = !{!"exec"}
diff --git a/test/CodeGen/AMDGPU/xnor.ll b/test/CodeGen/AMDGPU/xnor.ll
new file mode 100644
index 0000000000000..3991e615599db
--- /dev/null
+++ b/test/CodeGen/AMDGPU/xnor.ll
@@ -0,0 +1,83 @@
+; RUN: llc -march=amdgcn -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX600 %s
+; RUN: llc -march=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX700 %s
+; RUN: llc -march=amdgcn -mcpu=gfx800 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX800 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX900 %s
+
+; GCN-LABEL: {{^}}scalar_xnor_i32_one_use
+; GCN: s_xnor_b32
+define amdgpu_kernel void @scalar_xnor_i32_one_use(
+    i32 addrspace(1)* %r0, i32 %a, i32 %b) {
+entry:
+  %xor = xor i32 %a, %b
+  %r0.val = xor i32 %xor, -1
+  store i32 %r0.val, i32 addrspace(1)* %r0
+  ret void
+}
+
+; GCN-LABEL: {{^}}scalar_xnor_i32_mul_use
+; GCN-NOT: s_xnor_b32
+; GCN: s_xor_b32
+; GCN: s_not_b32
+; GCN: s_add_i32
+define amdgpu_kernel void @scalar_xnor_i32_mul_use(
+    i32 addrspace(1)* %r0, i32 addrspace(1)* %r1, i32 %a, i32 %b) {
+entry:
+  %xor = xor i32 %a, %b
+  %r0.val = xor i32 %xor, -1
+  %r1.val = add i32 %xor, %a
+  store i32 %r0.val, i32 addrspace(1)* %r0
+  store i32 %r1.val, i32 addrspace(1)* %r1
+  ret void
+}
+
+; GCN-LABEL: {{^}}scalar_xnor_i64_one_use
+; GCN: s_xnor_b64
+define amdgpu_kernel void @scalar_xnor_i64_one_use(
+    i64 addrspace(1)* %r0, i64 %a, i64 %b) {
+entry:
+  %xor = xor i64 %a, %b
+  %r0.val = xor i64 %xor, -1
+  store i64 %r0.val, i64 addrspace(1)* %r0
+  ret void
+}
+
+; GCN-LABEL: {{^}}scalar_xnor_i64_mul_use
+; GCN-NOT: s_xnor_b64
+; GCN: s_xor_b64
+; GCN: s_not_b64
+; GCN: s_add_u32
+; GCN: s_addc_u32
+define amdgpu_kernel void @scalar_xnor_i64_mul_use(
+    i64 addrspace(1)* %r0, i64 addrspace(1)* %r1, i64 %a, i64 %b) {
+entry:
+  %xor = xor i64 %a, %b
+  %r0.val = xor i64 %xor, -1
+  %r1.val = add i64 %xor, %a
+  store i64 %r0.val, i64 addrspace(1)* %r0
+  store i64 %r1.val, i64 addrspace(1)* %r1
+  ret void
+}
+
+; GCN-LABEL: {{^}}vector_xnor_i32_one_use
+; GCN-NOT: s_xnor_b32
+; GCN: v_xor_b32
+; GCN: v_not_b32
+define i32 @vector_xnor_i32_one_use(i32 %a, i32 %b) {
+entry:
+  %xor = xor i32 %a, %b
+  %r = xor i32 %xor, -1
+  ret i32 %r
+}
+
+; GCN-LABEL: {{^}}vector_xnor_i64_one_use
+; GCN-NOT: s_xnor_b64
+; GCN: v_xor_b32
+; GCN: v_xor_b32
+; GCN: v_not_b32
+; GCN: v_not_b32
+define i64 @vector_xnor_i64_one_use(i64 %a, i64 %b) {
+entry:
+  %xor = xor i64 %a, %b
+  %r = xor i64 %xor, -1
+  ret i64 %r
+}
diff --git a/test/CodeGen/AMDGPU/zext-lid.ll b/test/CodeGen/AMDGPU/zext-lid.ll
index 066f29277270a..9a9c1fe755007 100644
--- a/test/CodeGen/AMDGPU/zext-lid.ll
+++ b/test/CodeGen/AMDGPU/zext-lid.ll
@@ -63,6 +63,26 @@ bb:
   ret void
 }
 
+; OPT-LABEL: @func_test_workitem_id_x_known_max_range(
+; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !0
+define void @func_test_workitem_id_x_known_max_range(i32 addrspace(1)* nocapture %out) #0 {
+entry:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %and = and i32 %id, 1023
+  store i32 %and, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+; OPT-LABEL: @func_test_workitem_id_x_default_range(
+; OPT: tail call i32 @llvm.amdgcn.workitem.id.x(), !range !6
+define void @func_test_workitem_id_x_default_range(i32 addrspace(1)* nocapture %out) #4 {
+entry:
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %and = and i32 %id, 1023
+  store i32 %and, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #2
 
 declare i32 @llvm.amdgcn.workitem.id.y() #2
@@ -73,6 +93,7 @@ attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,128" }
 attributes #1 = { nounwind "amdgpu-flat-work-group-size"="512,512" }
 attributes #2 = { nounwind readnone speculatable }
 attributes #3 = { nounwind readnone }
+attributes #4 = { nounwind }
 
 !0 = !{i32 32, i32 4, i32 1}
 
@@ -82,3 +103,4 @@ attributes #3 = { nounwind readnone }
 ; OPT: !3 = !{i32 0, i32 4}
 ; OPT: !4 = !{i32 0, i32 1}
 ; OPT: !5 = !{i32 0, i32 512}
+; OPT: !6 = !{i32 0, i32 1024}
author	Dimitry Andric <dim@FreeBSD.org>	2017-12-18 20:10:56 +0000
committer	Dimitry Andric <dim@FreeBSD.org>	2017-12-18 20:10:56 +0000
commit	044eb2f6afba375a914ac9d8024f8f5142bb912e (patch)
tree	1475247dc9f9fe5be155ebd4c9069c75aadf8c20 /test/CodeGen/AMDGPU
parent	eb70dddbd77e120e5d490bd8fbe7ff3f8fa81c6b (diff)