diff options
Diffstat (limited to 'test/CodeGen/AMDGPU')
373 files changed, 27932 insertions, 5005 deletions
diff --git a/test/CodeGen/AMDGPU/32-bit-local-address-space.ll b/test/CodeGen/AMDGPU/32-bit-local-address-space.ll index ff8c90457876..6495a1480df6 100644 --- a/test/CodeGen/AMDGPU/32-bit-local-address-space.ll +++ b/test/CodeGen/AMDGPU/32-bit-local-address-space.ll @@ -57,8 +57,8 @@ entry: } ; FUNC-LABEL: {{^}}null_32bit_lds_ptr: -; SI: v_cmp_ne_i32 -; SI-NOT: v_cmp_ne_i32 +; SI: v_cmp_ne_u32 +; SI-NOT: v_cmp_ne_u32 ; SI: v_cndmask_b32 define void @null_32bit_lds_ptr(i32 addrspace(1)* %out, i32 addrspace(3)* %lds) nounwind { %cmp = icmp ne i32 addrspace(3)* %lds, null diff --git a/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll b/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll index 62b09dfedf15..5b7800996133 100644 --- a/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll +++ b/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll @@ -5,7 +5,7 @@ ; Tests for add. ; CHECK: name: addi32 -; CHECK: G_ADD i32 +; CHECK: {{%[0-9]+}}(s32) = G_ADD define i32 @addi32(i32 %arg1, i32 %arg2) { %res = add i32 %arg1, %arg2 ret i32 %res diff --git a/test/CodeGen/AMDGPU/add.i16.ll b/test/CodeGen/AMDGPU/add.i16.ll new file mode 100644 index 000000000000..a41d3071377d --- /dev/null +++ b/test/CodeGen/AMDGPU/add.i16.ll @@ -0,0 +1,149 @@ +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_add_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = add i16 %a, %b + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_constant: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0x7b, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_add_i16_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = add i16 %a, 123 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_neg_constant: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xfffffcb3, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_add_i16_neg_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = add i16 %a, -845 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_inline_neg1: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], -1, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_add_i16_inline_neg1(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = add i16 %a, -1 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_zext_to_i32: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: buffer_store_dword [[ADD]] +define void @v_test_add_i16_zext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = add i16 %a, %b + %ext = zext i16 %add to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_zext_to_i64: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI-DAG: v_add_u16_e32 v[[ADD:[0-9]+]], [[B]], [[A]] +; VI-DAG: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0 +; VI: buffer_store_dwordx2 v{{\[}}[[ADD]]:[[VZERO]]{{\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0{{$}} +define void @v_test_add_i16_zext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i64, i64 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = add i16 %a, %b + %ext = zext i16 %add to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i32: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16 +; VI-NEXT: buffer_store_dword [[SEXT]] +define void @v_test_add_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load i16, i16 addrspace(1)* %gep.in0 + %b = load i16, i16 addrspace(1)* %gep.in1 + %add = add i16 %a, %b + %ext = sext i16 %add to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_add_i16_sext_to_i64: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16 +; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] +; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @v_test_add_i16_sext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i64, i64 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load i16, i16 addrspace(1)* %gep.in0 + %b = load i16, i16 addrspace(1)* %gep.in1 + %add = add i16 %a, %b + %ext = sext i16 %add to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #0 + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/add_i128.ll b/test/CodeGen/AMDGPU/add_i128.ll new file mode 100644 index 000000000000..c80157ca9c58 --- /dev/null +++ b/test/CodeGen/AMDGPU/add_i128.ll @@ -0,0 +1,56 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}test_i128_vreg: +; GCN: v_add_i32_e32 v[[LO:[0-9]+]], vcc, +; GCN-NEXT: v_addc_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc +; GCN-NEXT: v_addc_u32_e32 v{{[0-9]+}}, vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc +; GCN-NEXT: v_addc_u32_e32 v[[HI:[0-9]+]], vcc, v{{[0-9]+}}, v{{[0-9]+}}, vcc +; GCN: buffer_store_dwordx4 v{{\[}}[[LO]]:[[HI]]], +define void @test_i128_vreg(i128 addrspace(1)* noalias %out, i128 addrspace(1)* noalias %inA, i128 addrspace(1)* noalias %inB) { + %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone + %a_ptr = getelementptr i128, i128 addrspace(1)* %inA, i32 %tid + %b_ptr = getelementptr i128, i128 addrspace(1)* %inB, i32 %tid + %a = load i128, i128 addrspace(1)* %a_ptr + %b = load i128, i128 addrspace(1)* %b_ptr + %result = add i128 %a, %b + store i128 %result, i128 addrspace(1)* %out + ret void +} + +; Check that the SGPR add operand is correctly moved to a VGPR. +; GCN-LABEL: {{^}}sgpr_operand: +; GCN: v_add_i32 +; GCN: v_addc_u32 +; GCN: v_addc_u32 +; GCN: v_addc_u32 +define void @sgpr_operand(i128 addrspace(1)* noalias %out, i128 addrspace(1)* noalias %in, i128 %a) { + %foo = load i128, i128 addrspace(1)* %in, align 8 + %result = add i128 %foo, %a + store i128 %result, i128 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sgpr_operand_reversed: +; GCN: v_add_i32 +; GCN: v_addc_u32 +; GCN: v_addc_u32 +; GCN: v_addc_u32 +define void @sgpr_operand_reversed(i128 addrspace(1)* noalias %out, i128 addrspace(1)* noalias %in, i128 %a) { + %foo = load i128, i128 addrspace(1)* %in, align 8 + %result = add i128 %a, %foo + store i128 %result, i128 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_sreg: +; GCN: s_add_u32 +; GCN: s_addc_u32 +; GCN: s_addc_u32 +; GCN: s_addc_u32 +define void @test_sreg(i128 addrspace(1)* noalias %out, i128 %a, i128 %b) { + %result = add i128 %a, %b + store i128 %result, i128 addrspace(1)* %out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() readnone diff --git a/test/CodeGen/AMDGPU/addrspacecast.ll b/test/CodeGen/AMDGPU/addrspacecast.ll index 5a173e954f8d..0a2130c96add 100644 --- a/test/CodeGen/AMDGPU/addrspacecast.ll +++ b/test/CodeGen/AMDGPU/addrspacecast.ll @@ -11,7 +11,7 @@ ; HSA-DAG: v_mov_b32_e32 [[VAPERTURE:v[0-9]+]], [[APERTURE]] ; HSA-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]] -; HSA-DAG: v_cmp_ne_i32_e64 vcc, -1, [[PTR]] +; HSA-DAG: v_cmp_ne_u32_e64 vcc, [[PTR]], -1 ; HSA-DAG: v_cndmask_b32_e32 v[[HI:[0-9]+]], 0, [[VAPERTURE]] ; HSA-DAG: v_cndmask_b32_e32 v[[LO:[0-9]+]], 0, [[VPTR]] ; HSA-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7 @@ -34,7 +34,7 @@ define void @use_group_to_flat_addrspacecast(i32 addrspace(3)* %ptr) #0 { ; HSA-DAG: v_mov_b32_e32 [[VAPERTURE:v[0-9]+]], [[APERTURE]] ; HSA-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]] -; HSA-DAG: v_cmp_ne_i32_e64 vcc, -1, [[PTR]] +; HSA-DAG: v_cmp_ne_u32_e64 vcc, [[PTR]], -1 ; HSA-DAG: v_cndmask_b32_e32 v[[HI:[0-9]+]], 0, [[VAPERTURE]] ; HSA-DAG: v_cndmask_b32_e32 v[[LO:[0-9]+]], 0, [[VPTR]] ; HSA-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7 @@ -79,7 +79,7 @@ define void @use_constant_to_flat_addrspacecast(i32 addrspace(2)* %ptr) #0 { ; HSA: enable_sgpr_queue_ptr = 0 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}} -; HSA-DAG: v_cmp_ne_i64_e64 vcc, 0, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}} +; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}} ; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]] ; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]] ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}} @@ -96,7 +96,7 @@ define void @use_flat_to_group_addrspacecast(i32 addrspace(4)* %ptr) #0 { ; HSA: enable_sgpr_queue_ptr = 0 ; HSA: s_load_dwordx2 s{{\[}}[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]{{\]}} -; HSA-DAG: v_cmp_ne_i64_e64 vcc, 0, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}} +; HSA-DAG: v_cmp_ne_u64_e64 vcc, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0{{$}} ; HSA-DAG: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], s[[PTR_LO]] ; HSA-DAG: v_cndmask_b32_e32 [[CASTPTR:v[0-9]+]], -1, v[[VPTR_LO]] ; HSA-DAG: v_mov_b32_e32 v[[K:[0-9]+]], 0{{$}} diff --git a/test/CodeGen/AMDGPU/bitcast.ll b/test/CodeGen/AMDGPU/amdgcn.bitcast.ll index 87ef5978ebfc..87ef5978ebfc 100644 --- a/test/CodeGen/AMDGPU/bitcast.ll +++ b/test/CodeGen/AMDGPU/amdgcn.bitcast.ll diff --git a/test/CodeGen/AMDGPU/amdgcn.work-item-intrinsics.ll b/test/CodeGen/AMDGPU/amdgcn.work-item-intrinsics.ll deleted file mode 100644 index b1b3b9930d1f..000000000000 --- a/test/CodeGen/AMDGPU/amdgcn.work-item-intrinsics.ll +++ /dev/null @@ -1,114 +0,0 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=SI-NOHSA -check-prefix=GCN-NOHSA -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-NOHSA -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s - - -; FUNC-LABEL: {{^}}workdim: - -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0xb -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x2c -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NOHSA: buffer_store_dword [[VVAL]] - -define void @workdim (i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.read.workdim() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; The workgroup.id values are stored in sgprs offset by the number of user -; sgprs. - -; FUNC-LABEL: {{^}}workgroup_id_x: -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], s2{{$}} -; GCN-NOHSA: buffer_store_dword [[VVAL]] - -; GCN-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 -; GCN: COMPUTE_PGM_RSRC2:TGID_X_EN: 1 -; GCN: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 -; GCN: COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 -; GCN: COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0 -define void @workgroup_id_x(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workgroup.id.x() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; FUNC-LABEL: {{^}}workgroup_id_y: -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], s3 -; GCN-NOHSA: buffer_store_dword [[VVAL]] - -; GCN-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 -define void @workgroup_id_y(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workgroup.id.y() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; FUNC-LABEL: {{^}}workgroup_id_z: -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], s3{{$}} -; GCN-NOHSA: buffer_store_dword [[VVAL]] - -; GCN-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 -; GCN: COMPUTE_PGM_RSRC2:TGID_X_EN: 1 -; GCN: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 -; GCN: COMPUTE_PGM_RSRC2:TGID_Z_EN: 1 -; GCN: COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 0 -define void @workgroup_id_z(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workgroup.id.z() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; GCN-NOHSA: .section .AMDGPU.config -; GCN-NOHSA: .long 47180 -; GCN-NOHSA-NEXT: .long 132{{$}} - -; FUNC-LABEL: {{^}}workitem_id_x: -; GCN-NOHSA: buffer_store_dword v0 -define void @workitem_id_x(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workitem.id.x() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; GCN-NOHSA: .section .AMDGPU.config -; GCN-NOHSA: .long 47180 -; GCN-NOHSA-NEXT: .long 2180{{$}} - -; FUNC-LABEL: {{^}}workitem_id_y: - -; GCN-NOHSA: buffer_store_dword v1 -define void @workitem_id_y(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workitem.id.y() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; GCN-NOHSA: .section .AMDGPU.config -; GCN-NOHSA: .long 47180 -; GCN-NOHSA-NEXT: .long 4228{{$}} - -; FUNC-LABEL: {{^}}workitem_id_z: -; GCN-NOHSA: buffer_store_dword v2 -define void @workitem_id_z(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.workitem.id.z() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -declare i32 @llvm.amdgcn.workgroup.id.x() #0 -declare i32 @llvm.amdgcn.workgroup.id.y() #0 -declare i32 @llvm.amdgcn.workgroup.id.z() #0 - -declare i32 @llvm.amdgcn.workitem.id.x() #0 -declare i32 @llvm.amdgcn.workitem.id.y() #0 -declare i32 @llvm.amdgcn.workitem.id.z() #0 - -declare i32 @llvm.amdgcn.read.workdim() #0 diff --git a/test/CodeGen/AMDGPU/amdgpu-codegenprepare.ll b/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll index d78c75165be2..d78c75165be2 100644 --- a/test/CodeGen/AMDGPU/amdgpu-codegenprepare.ll +++ b/test/CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll diff --git a/test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll b/test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll new file mode 100644 index 000000000000..13e4192ccd72 --- /dev/null +++ b/test/CodeGen/AMDGPU/amdgpu-codegenprepare-i16-to-i32.ll @@ -0,0 +1,2106 @@ +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: opt -S -mtriple=amdgcn-- -mcpu=tonga -amdgpu-codegenprepare %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: @add_i3( +; SI: %r = add i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @add_i3(i3 %a, i3 %b) { + %r = add i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @add_nsw_i3( +; SI: %r = add nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @add_nsw_i3(i3 %a, i3 %b) { + %r = add nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @add_nuw_i3( +; SI: %r = add nuw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @add_nuw_i3(i3 %a, i3 %b) { + %r = add nuw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @add_nuw_nsw_i3( +; SI: %r = add nuw nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @add_nuw_nsw_i3(i3 %a, i3 %b) { + %r = add nuw nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @sub_i3( +; SI: %r = sub i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @sub_i3(i3 %a, i3 %b) { + %r = sub i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @sub_nsw_i3( +; SI: %r = sub nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @sub_nsw_i3(i3 %a, i3 %b) { + %r = sub nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @sub_nuw_i3( +; SI: %r = sub nuw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @sub_nuw_i3(i3 %a, i3 %b) { + %r = sub nuw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @sub_nuw_nsw_i3( +; SI: %r = sub nuw nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @sub_nuw_nsw_i3(i3 %a, i3 %b) { + %r = sub nuw nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @mul_i3( +; SI: %r = mul i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @mul_i3(i3 %a, i3 %b) { + %r = mul i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @mul_nsw_i3( +; SI: %r = mul nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @mul_nsw_i3(i3 %a, i3 %b) { + %r = mul nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @mul_nuw_i3( +; SI: %r = mul nuw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @mul_nuw_i3(i3 %a, i3 %b) { + %r = mul nuw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @mul_nuw_nsw_i3( +; SI: %r = mul nuw nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @mul_nuw_nsw_i3(i3 %a, i3 %b) { + %r = mul nuw nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @urem_i3( +; SI: %r = urem i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = urem i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @urem_i3(i3 %a, i3 %b) { + %r = urem i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @srem_i3( +; SI: %r = srem i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = srem i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @srem_i3(i3 %a, i3 %b) { + %r = srem i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @shl_i3( +; SI: %r = shl i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @shl_i3(i3 %a, i3 %b) { + %r = shl i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @shl_nsw_i3( +; SI: %r = shl nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @shl_nsw_i3(i3 %a, i3 %b) { + %r = shl nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @shl_nuw_i3( +; SI: %r = shl nuw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @shl_nuw_i3(i3 %a, i3 %b) { + %r = shl nuw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @shl_nuw_nsw_i3( +; SI: %r = shl nuw nsw i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @shl_nuw_nsw_i3(i3 %a, i3 %b) { + %r = shl nuw nsw i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @lshr_i3( +; SI: %r = lshr i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = lshr i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @lshr_i3(i3 %a, i3 %b) { + %r = lshr i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @lshr_exact_i3( +; SI: %r = lshr exact i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = lshr exact i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @lshr_exact_i3(i3 %a, i3 %b) { + %r = lshr exact i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @ashr_i3( +; SI: %r = ashr i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = ashr i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @ashr_i3(i3 %a, i3 %b) { + %r = ashr i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @ashr_exact_i3( +; SI: %r = ashr exact i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = ashr exact i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @ashr_exact_i3(i3 %a, i3 %b) { + %r = ashr exact i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @and_i3( +; SI: %r = and i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = and i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @and_i3(i3 %a, i3 %b) { + %r = and i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @or_i3( +; SI: %r = or i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = or i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @or_i3(i3 %a, i3 %b) { + %r = or i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @xor_i3( +; SI: %r = xor i3 %a, %b +; SI-NEXT: ret i3 %r +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = xor i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[R_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @xor_i3(i3 %a, i3 %b) { + %r = xor i3 %a, %b + ret i3 %r +} + +; GCN-LABEL: @select_eq_i3( +; SI: %cmp = icmp eq i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp eq i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_eq_i3(i3 %a, i3 %b) { + %cmp = icmp eq i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_ne_i3( +; SI: %cmp = icmp ne i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ne i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_ne_i3(i3 %a, i3 %b) { + %cmp = icmp ne i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_ugt_i3( +; SI: %cmp = icmp ugt i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ugt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_ugt_i3(i3 %a, i3 %b) { + %cmp = icmp ugt i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_uge_i3( +; SI: %cmp = icmp uge i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp uge i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_uge_i3(i3 %a, i3 %b) { + %cmp = icmp uge i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_ult_i3( +; SI: %cmp = icmp ult i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ult i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_ult_i3(i3 %a, i3 %b) { + %cmp = icmp ult i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_ule_i3( +; SI: %cmp = icmp ule i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ule i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_ule_i3(i3 %a, i3 %b) { + %cmp = icmp ule i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_sgt_i3( +; SI: %cmp = icmp sgt i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sgt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_sgt_i3(i3 %a, i3 %b) { + %cmp = icmp sgt i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_sge_i3( +; SI: %cmp = icmp sge i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sge i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_sge_i3(i3 %a, i3 %b) { + %cmp = icmp sge i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_slt_i3( +; SI: %cmp = icmp slt i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp slt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_slt_i3(i3 %a, i3 %b) { + %cmp = icmp slt i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +; GCN-LABEL: @select_sle_i3( +; SI: %cmp = icmp sle i3 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i3 %a, i3 %b +; SI-NEXT: ret i3 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sle i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i3 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i3 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_3:[0-9]+]] = trunc i32 %[[SEL_32]] to i3 +; VI-NEXT: ret i3 %[[SEL_3]] +define i3 @select_sle_i3(i3 %a, i3 %b) { + %cmp = icmp sle i3 %a, %b + %sel = select i1 %cmp, i3 %a, i3 %b + ret i3 %sel +} + +declare i3 @llvm.bitreverse.i3(i3) +; GCN-LABEL: @bitreverse_i3( +; SI: %brev = call i3 @llvm.bitreverse.i3(i3 %a) +; SI-NEXT: ret i3 %brev +; VI: %[[A_32:[0-9]+]] = zext i3 %a to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = call i32 @llvm.bitreverse.i32(i32 %[[A_32]]) +; VI-NEXT: %[[S_32:[0-9]+]] = lshr i32 %[[R_32]], 29 +; VI-NEXT: %[[R_3:[0-9]+]] = trunc i32 %[[S_32]] to i3 +; VI-NEXT: ret i3 %[[R_3]] +define i3 @bitreverse_i3(i3 %a) { + %brev = call i3 @llvm.bitreverse.i3(i3 %a) + ret i3 %brev +} + +; GCN-LABEL: @add_i16( +; SI: %r = add i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @add_i16(i16 %a, i16 %b) { + %r = add i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @constant_add_i16( +; VI: ret i16 3 +define i16 @constant_add_i16() { + %r = add i16 1, 2 + ret i16 %r +} + +; GCN-LABEL: @constant_add_nsw_i16( +; VI: ret i16 3 +define i16 @constant_add_nsw_i16() { + %r = add nsw i16 1, 2 + ret i16 %r +} + +; GCN-LABEL: @constant_add_nuw_i16( +; VI: ret i16 3 +define i16 @constant_add_nuw_i16() { + %r = add nsw i16 1, 2 + ret i16 %r +} + +; GCN-LABEL: @add_nsw_i16( +; SI: %r = add nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @add_nsw_i16(i16 %a, i16 %b) { + %r = add nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @add_nuw_i16( +; SI: %r = add nuw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @add_nuw_i16(i16 %a, i16 %b) { + %r = add nuw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @add_nuw_nsw_i16( +; SI: %r = add nuw nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @add_nuw_nsw_i16(i16 %a, i16 %b) { + %r = add nuw nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @sub_i16( +; SI: %r = sub i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @sub_i16(i16 %a, i16 %b) { + %r = sub i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @sub_nsw_i16( +; SI: %r = sub nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @sub_nsw_i16(i16 %a, i16 %b) { + %r = sub nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @sub_nuw_i16( +; SI: %r = sub nuw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @sub_nuw_i16(i16 %a, i16 %b) { + %r = sub nuw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @sub_nuw_nsw_i16( +; SI: %r = sub nuw nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @sub_nuw_nsw_i16(i16 %a, i16 %b) { + %r = sub nuw nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @mul_i16( +; SI: %r = mul i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @mul_i16(i16 %a, i16 %b) { + %r = mul i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @mul_nsw_i16( +; SI: %r = mul nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @mul_nsw_i16(i16 %a, i16 %b) { + %r = mul nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @mul_nuw_i16( +; SI: %r = mul nuw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @mul_nuw_i16(i16 %a, i16 %b) { + %r = mul nuw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @mul_nuw_nsw_i16( +; SI: %r = mul nuw nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @mul_nuw_nsw_i16(i16 %a, i16 %b) { + %r = mul nuw nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @urem_i16( +; SI: %r = urem i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = urem i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @urem_i16(i16 %a, i16 %b) { + %r = urem i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @srem_i16( +; SI: %r = srem i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = srem i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @srem_i16(i16 %a, i16 %b) { + %r = srem i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @shl_i16( +; SI: %r = shl i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @shl_i16(i16 %a, i16 %b) { + %r = shl i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @shl_nsw_i16( +; SI: %r = shl nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @shl_nsw_i16(i16 %a, i16 %b) { + %r = shl nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @shl_nuw_i16( +; SI: %r = shl nuw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @shl_nuw_i16(i16 %a, i16 %b) { + %r = shl nuw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @shl_nuw_nsw_i16( +; SI: %r = shl nuw nsw i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw nsw i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @shl_nuw_nsw_i16(i16 %a, i16 %b) { + %r = shl nuw nsw i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @lshr_i16( +; SI: %r = lshr i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = lshr i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @lshr_i16(i16 %a, i16 %b) { + %r = lshr i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @lshr_exact_i16( +; SI: %r = lshr exact i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = lshr exact i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @lshr_exact_i16(i16 %a, i16 %b) { + %r = lshr exact i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @ashr_i16( +; SI: %r = ashr i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = ashr i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @ashr_i16(i16 %a, i16 %b) { + %r = ashr i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @ashr_exact_i16( +; SI: %r = ashr exact i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = ashr exact i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @ashr_exact_i16(i16 %a, i16 %b) { + %r = ashr exact i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @constant_lshr_exact_i16( +; VI: ret i16 2 +define i16 @constant_lshr_exact_i16(i16 %a, i16 %b) { + %r = lshr exact i16 4, 1 + ret i16 %r +} + +; GCN-LABEL: @and_i16( +; SI: %r = and i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = and i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @and_i16(i16 %a, i16 %b) { + %r = and i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @or_i16( +; SI: %r = or i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = or i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @or_i16(i16 %a, i16 %b) { + %r = or i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @xor_i16( +; SI: %r = xor i16 %a, %b +; SI-NEXT: ret i16 %r +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = xor i32 %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[R_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @xor_i16(i16 %a, i16 %b) { + %r = xor i16 %a, %b + ret i16 %r +} + +; GCN-LABEL: @select_eq_i16( +; SI: %cmp = icmp eq i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp eq i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_eq_i16(i16 %a, i16 %b) { + %cmp = icmp eq i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_ne_i16( +; SI: %cmp = icmp ne i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ne i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_ne_i16(i16 %a, i16 %b) { + %cmp = icmp ne i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_ugt_i16( +; SI: %cmp = icmp ugt i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ugt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_ugt_i16(i16 %a, i16 %b) { + %cmp = icmp ugt i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_uge_i16( +; SI: %cmp = icmp uge i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp uge i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_uge_i16(i16 %a, i16 %b) { + %cmp = icmp uge i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_ult_i16( +; SI: %cmp = icmp ult i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ult i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_ult_i16(i16 %a, i16 %b) { + %cmp = icmp ult i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_ule_i16( +; SI: %cmp = icmp ule i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ule i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_ule_i16(i16 %a, i16 %b) { + %cmp = icmp ule i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_sgt_i16( +; SI: %cmp = icmp sgt i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sgt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_sgt_i16(i16 %a, i16 %b) { + %cmp = icmp sgt i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_sge_i16( +; SI: %cmp = icmp sge i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sge i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_sge_i16(i16 %a, i16 %b) { + %cmp = icmp sge i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_slt_i16( +; SI: %cmp = icmp slt i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp slt i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_slt_i16(i16 %a, i16 %b) { + %cmp = icmp slt i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +; GCN-LABEL: @select_sle_i16( +; SI: %cmp = icmp sle i16 %a, %b +; SI-NEXT: %sel = select i1 %cmp, i16 %a, i16 %b +; SI-NEXT: ret i16 %sel +; VI: %[[A_32_0:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sle i32 %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext i16 %a to i32 +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext i16 %b to i32 +; VI-NEXT: %[[SEL_32:[0-9]+]] = select i1 %[[CMP]], i32 %[[A_32_1]], i32 %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc i32 %[[SEL_32]] to i16 +; VI-NEXT: ret i16 %[[SEL_16]] +define i16 @select_sle_i16(i16 %a, i16 %b) { + %cmp = icmp sle i16 %a, %b + %sel = select i1 %cmp, i16 %a, i16 %b + ret i16 %sel +} + +declare i16 @llvm.bitreverse.i16(i16) +; GCN-LABEL: @bitreverse_i16( +; SI: %brev = call i16 @llvm.bitreverse.i16(i16 %a) +; SI-NEXT: ret i16 %brev +; VI: %[[A_32:[0-9]+]] = zext i16 %a to i32 +; VI-NEXT: %[[R_32:[0-9]+]] = call i32 @llvm.bitreverse.i32(i32 %[[A_32]]) +; VI-NEXT: %[[S_32:[0-9]+]] = lshr i32 %[[R_32]], 16 +; VI-NEXT: %[[R_16:[0-9]+]] = trunc i32 %[[S_32]] to i16 +; VI-NEXT: ret i16 %[[R_16]] +define i16 @bitreverse_i16(i16 %a) { + %brev = call i16 @llvm.bitreverse.i16(i16 %a) + ret i16 %brev +} + +; GCN-LABEL: @add_3xi15( +; SI: %r = add <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @add_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = add <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @add_nsw_3xi15( +; SI: %r = add nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @add_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = add nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @add_nuw_3xi15( +; SI: %r = add nuw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @add_nuw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = add nuw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @add_nuw_nsw_3xi15( +; SI: %r = add nuw nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @add_nuw_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = add nuw nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @sub_3xi15( +; SI: %r = sub <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @sub_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = sub <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @sub_nsw_3xi15( +; SI: %r = sub nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @sub_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = sub nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @sub_nuw_3xi15( +; SI: %r = sub nuw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @sub_nuw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = sub nuw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @sub_nuw_nsw_3xi15( +; SI: %r = sub nuw nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @sub_nuw_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = sub nuw nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @mul_3xi15( +; SI: %r = mul <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @mul_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = mul <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @mul_nsw_3xi15( +; SI: %r = mul nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @mul_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = mul nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @mul_nuw_3xi15( +; SI: %r = mul nuw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @mul_nuw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = mul nuw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @mul_nuw_nsw_3xi15( +; SI: %r = mul nuw nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @mul_nuw_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = mul nuw nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @urem_3xi15( +; SI: %r = urem <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = urem <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @urem_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = urem <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @srem_3xi15( +; SI: %r = srem <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = srem <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @srem_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = srem <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @shl_3xi15( +; SI: %r = shl <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @shl_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = shl <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @shl_nsw_3xi15( +; SI: %r = shl nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @shl_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = shl nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @shl_nuw_3xi15( +; SI: %r = shl nuw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @shl_nuw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = shl nuw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @shl_nuw_nsw_3xi15( +; SI: %r = shl nuw nsw <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @shl_nuw_nsw_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = shl nuw nsw <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @lshr_3xi15( +; SI: %r = lshr <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = lshr <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @lshr_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = lshr <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @lshr_exact_3xi15( +; SI: %r = lshr exact <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = lshr exact <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @lshr_exact_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = lshr exact <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @ashr_3xi15( +; SI: %r = ashr <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = ashr <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @ashr_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = ashr <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @ashr_exact_3xi15( +; SI: %r = ashr exact <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = ashr exact <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @ashr_exact_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = ashr exact <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @and_3xi15( +; SI: %r = and <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = and <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @and_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = and <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @or_3xi15( +; SI: %r = or <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = or <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @or_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = or <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @xor_3xi15( +; SI: %r = xor <3 x i15> %a, %b +; SI-NEXT: ret <3 x i15> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = xor <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @xor_3xi15(<3 x i15> %a, <3 x i15> %b) { + %r = xor <3 x i15> %a, %b + ret <3 x i15> %r +} + +; GCN-LABEL: @select_eq_3xi15( +; SI: %cmp = icmp eq <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp eq <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_eq_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp eq <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_ne_3xi15( +; SI: %cmp = icmp ne <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ne <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_ne_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp ne <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_ugt_3xi15( +; SI: %cmp = icmp ugt <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ugt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_ugt_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp ugt <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_uge_3xi15( +; SI: %cmp = icmp uge <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp uge <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_uge_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp uge <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_ult_3xi15( +; SI: %cmp = icmp ult <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ult <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_ult_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp ult <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_ule_3xi15( +; SI: %cmp = icmp ule <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ule <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_ule_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp ule <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_sgt_3xi15( +; SI: %cmp = icmp sgt <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sgt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_sgt_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp sgt <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_sge_3xi15( +; SI: %cmp = icmp sge <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sge <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_sge_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp sge <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_slt_3xi15( +; SI: %cmp = icmp slt <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp slt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_slt_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp slt <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +; GCN-LABEL: @select_sle_3xi15( +; SI: %cmp = icmp sle <3 x i15> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b +; SI-NEXT: ret <3 x i15> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sle <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i15> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_15:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[SEL_15]] +define <3 x i15> @select_sle_3xi15(<3 x i15> %a, <3 x i15> %b) { + %cmp = icmp sle <3 x i15> %a, %b + %sel = select <3 x i1> %cmp, <3 x i15> %a, <3 x i15> %b + ret <3 x i15> %sel +} + +declare <3 x i15> @llvm.bitreverse.v3i15(<3 x i15>) +; GCN-LABEL: @bitreverse_3xi15( +; SI: %brev = call <3 x i15> @llvm.bitreverse.v3i15(<3 x i15> %a) +; SI-NEXT: ret <3 x i15> %brev +; VI: %[[A_32:[0-9]+]] = zext <3 x i15> %a to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = call <3 x i32> @llvm.bitreverse.v3i32(<3 x i32> %[[A_32]]) +; VI-NEXT: %[[S_32:[0-9]+]] = lshr <3 x i32> %[[R_32]], <i32 17, i32 17, i32 17> +; VI-NEXT: %[[R_15:[0-9]+]] = trunc <3 x i32> %[[S_32]] to <3 x i15> +; VI-NEXT: ret <3 x i15> %[[R_15]] +define <3 x i15> @bitreverse_3xi15(<3 x i15> %a) { + %brev = call <3 x i15> @llvm.bitreverse.v3i15(<3 x i15> %a) + ret <3 x i15> %brev +} + +; GCN-LABEL: @add_3xi16( +; SI: %r = add <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @add_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = add <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @add_nsw_3xi16( +; SI: %r = add nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @add_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = add nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @add_nuw_3xi16( +; SI: %r = add nuw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @add_nuw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = add nuw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @add_nuw_nsw_3xi16( +; SI: %r = add nuw nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = add nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @add_nuw_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = add nuw nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @sub_3xi16( +; SI: %r = sub <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @sub_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = sub <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @sub_nsw_3xi16( +; SI: %r = sub nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @sub_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = sub nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @sub_nuw_3xi16( +; SI: %r = sub nuw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @sub_nuw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = sub nuw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @sub_nuw_nsw_3xi16( +; SI: %r = sub nuw nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = sub nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @sub_nuw_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = sub nuw nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @mul_3xi16( +; SI: %r = mul <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @mul_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = mul <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @mul_nsw_3xi16( +; SI: %r = mul nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @mul_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = mul nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @mul_nuw_3xi16( +; SI: %r = mul nuw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @mul_nuw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = mul nuw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @mul_nuw_nsw_3xi16( +; SI: %r = mul nuw nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = mul nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @mul_nuw_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = mul nuw nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @urem_3xi16( +; SI: %r = urem <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = urem <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @urem_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = urem <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @srem_3xi16( +; SI: %r = srem <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = srem <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @srem_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = srem <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @shl_3xi16( +; SI: %r = shl <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @shl_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = shl <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @shl_nsw_3xi16( +; SI: %r = shl nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @shl_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = shl nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @shl_nuw_3xi16( +; SI: %r = shl nuw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @shl_nuw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = shl nuw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @shl_nuw_nsw_3xi16( +; SI: %r = shl nuw nsw <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = shl nuw nsw <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @shl_nuw_nsw_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = shl nuw nsw <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @lshr_3xi16( +; SI: %r = lshr <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = lshr <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @lshr_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = lshr <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @lshr_exact_3xi16( +; SI: %r = lshr exact <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = lshr exact <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @lshr_exact_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = lshr exact <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @ashr_3xi16( +; SI: %r = ashr <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = ashr <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @ashr_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = ashr <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @ashr_exact_3xi16( +; SI: %r = ashr exact <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = ashr exact <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @ashr_exact_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = ashr exact <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @and_3xi16( +; SI: %r = and <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = and <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @and_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = and <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @or_3xi16( +; SI: %r = or <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = or <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @or_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = or <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @xor_3xi16( +; SI: %r = xor <3 x i16> %a, %b +; SI-NEXT: ret <3 x i16> %r +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = xor <3 x i32> %[[A_32]], %[[B_32]] +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[R_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @xor_3xi16(<3 x i16> %a, <3 x i16> %b) { + %r = xor <3 x i16> %a, %b + ret <3 x i16> %r +} + +; GCN-LABEL: @select_eq_3xi16( +; SI: %cmp = icmp eq <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp eq <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_eq_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp eq <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_ne_3xi16( +; SI: %cmp = icmp ne <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ne <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_ne_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp ne <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_ugt_3xi16( +; SI: %cmp = icmp ugt <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ugt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_ugt_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp ugt <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_uge_3xi16( +; SI: %cmp = icmp uge <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp uge <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_uge_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp uge <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_ult_3xi16( +; SI: %cmp = icmp ult <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ult <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_ult_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp ult <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_ule_3xi16( +; SI: %cmp = icmp ule <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp ule <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = zext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_ule_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp ule <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_sgt_3xi16( +; SI: %cmp = icmp sgt <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sgt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_sgt_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp sgt <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_sge_3xi16( +; SI: %cmp = icmp sge <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sge <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_sge_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp sge <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_slt_3xi16( +; SI: %cmp = icmp slt <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp slt <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_slt_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp slt <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +; GCN-LABEL: @select_sle_3xi16( +; SI: %cmp = icmp sle <3 x i16> %a, %b +; SI-NEXT: %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b +; SI-NEXT: ret <3 x i16> %sel +; VI: %[[A_32_0:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_0:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[CMP:[0-9]+]] = icmp sle <3 x i32> %[[A_32_0]], %[[B_32_0]] +; VI-NEXT: %[[A_32_1:[0-9]+]] = sext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[B_32_1:[0-9]+]] = sext <3 x i16> %b to <3 x i32> +; VI-NEXT: %[[SEL_32:[0-9]+]] = select <3 x i1> %[[CMP]], <3 x i32> %[[A_32_1]], <3 x i32> %[[B_32_1]] +; VI-NEXT: %[[SEL_16:[0-9]+]] = trunc <3 x i32> %[[SEL_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[SEL_16]] +define <3 x i16> @select_sle_3xi16(<3 x i16> %a, <3 x i16> %b) { + %cmp = icmp sle <3 x i16> %a, %b + %sel = select <3 x i1> %cmp, <3 x i16> %a, <3 x i16> %b + ret <3 x i16> %sel +} + +declare <3 x i16> @llvm.bitreverse.v3i16(<3 x i16>) +; GCN-LABEL: @bitreverse_3xi16( +; SI: %brev = call <3 x i16> @llvm.bitreverse.v3i16(<3 x i16> %a) +; SI-NEXT: ret <3 x i16> %brev +; VI: %[[A_32:[0-9]+]] = zext <3 x i16> %a to <3 x i32> +; VI-NEXT: %[[R_32:[0-9]+]] = call <3 x i32> @llvm.bitreverse.v3i32(<3 x i32> %[[A_32]]) +; VI-NEXT: %[[S_32:[0-9]+]] = lshr <3 x i32> %[[R_32]], <i32 16, i32 16, i32 16> +; VI-NEXT: %[[R_16:[0-9]+]] = trunc <3 x i32> %[[S_32]] to <3 x i16> +; VI-NEXT: ret <3 x i16> %[[R_16]] +define <3 x i16> @bitreverse_3xi16(<3 x i16> %a) { + %brev = call <3 x i16> @llvm.bitreverse.v3i16(<3 x i16> %a) + ret <3 x i16> %brev +} diff --git a/test/CodeGen/AMDGPU/amdgpu.private-memory.ll b/test/CodeGen/AMDGPU/amdgpu.private-memory.ll index bd0817d30413..1511e1343808 100644 --- a/test/CodeGen/AMDGPU/amdgpu.private-memory.ll +++ b/test/CodeGen/AMDGPU/amdgpu.private-memory.ll @@ -1,9 +1,9 @@ -; RUN: llc -show-mc-encoding -mattr=+promote-alloca -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -; RUN: llc -show-mc-encoding -mattr=+promote-alloca -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE -; RUN: llc -show-mc-encoding -mattr=-promote-alloca -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -; RUN: llc -show-mc-encoding -mattr=-promote-alloca -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA -; RUN: llc -show-mc-encoding -mattr=+promote-alloca -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -; RUN: llc -show-mc-encoding -mattr=-promote-alloca -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC +; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC +; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn--amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-PROMOTE +; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -march=amdgcn < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC +; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -mcpu=kaveri -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC -check-prefix=HSA-ALLOCA +; RUN: llc -show-mc-encoding -mattr=+promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-PROMOTE -check-prefix=SI -check-prefix=FUNC +; RUN: llc -show-mc-encoding -mattr=-promote-alloca -amdgpu-load-store-vectorizer=0 -verify-machineinstrs -mtriple=amdgcn-amdhsa -march=amdgcn -mcpu=tonga -mattr=-unaligned-buffer-access < %s | FileCheck %s -check-prefix=SI-ALLOCA -check-prefix=SI -check-prefix=FUNC ; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=HSAOPT -check-prefix=OPT %s ; RUN: opt -S -mtriple=amdgcn-unknown-unknown -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck -check-prefix=NOHSAOPT -check-prefix=OPT %s @@ -227,9 +227,10 @@ for.end: ; R600: MOVA_INT -; SI-PROMOTE-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen ; encoding: [0x00,0x10,0x68,0xe0 -; SI-PROMOTE-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:2 ; encoding: [0x02,0x10,0x68,0xe0 -; SI-PROMOTE: buffer_load_sshort v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} +; SI-PROMOTE-DAG: buffer_store_short v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} ; encoding: [0x00,0x00,0x68,0xe0 +; SI-PROMOTE-DAG: buffer_store_short v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:2 ; encoding: [0x02,0x00,0x68,0xe0 +; Loaded value is 0 or 1, so sext will become zext, so we get buffer_load_ushort instead of buffer_load_sshort. +; SI-PROMOTE: buffer_load_ushort v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} define void @short_array(i32 addrspace(1)* %out, i32 %index) #0 { entry: %0 = alloca [2 x i16] @@ -248,8 +249,11 @@ entry: ; R600: MOVA_INT -; SI-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen ; encoding: [0x00,0x10,0x60,0xe0 -; SI-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:1 ; encoding: [0x01,0x10,0x60,0xe0 +; SI-PROMOTE-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} ; encoding: +; SI-PROMOTE-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:1 ; encoding: + +; SI-ALLOCA-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} ; encoding: [0x00,0x00,0x60,0xe0 +; SI-ALLOCA-DAG: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:1 ; encoding: [0x01,0x00,0x60,0xe0 define void @char_array(i32 addrspace(1)* %out, i32 %index) #0 { entry: %0 = alloca [2 x i8] @@ -262,16 +266,17 @@ entry: %5 = sext i8 %4 to i32 store i32 %5, i32 addrspace(1)* %out ret void - } ; Test that two stack objects are not stored in the same register ; The second stack object should be in T3.X ; FUNC-LABEL: {{^}}no_overlap: -; R600_CHECK: MOV -; R600_CHECK: [[CHAN:[XYZW]]]+ +; R600-CHECK: MOV +; R600-CHECK: [[CHAN:[XYZW]]]+ ; R600-NOT: [[CHAN]]+ -; SI: v_mov_b32_e32 v3 +; +; A total of 5 bytes should be allocated and used. +; SI: buffer_store_byte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:4 ; define void @no_overlap(i32 addrspace(1)* %out, i32 %in) #0 { entry: %0 = alloca [3 x i8], align 1 @@ -545,7 +550,7 @@ entry: ret void } -attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="2" } +attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" } ; HSAOPT: !0 = !{} ; HSAOPT: !1 = !{i32 0, i32 2048} diff --git a/test/CodeGen/AMDGPU/amdgpu.work-item-intrinsics.deprecated.ll b/test/CodeGen/AMDGPU/amdgpu.work-item-intrinsics.deprecated.ll index 853788b92aae..f05f027fb3c8 100644 --- a/test/CodeGen/AMDGPU/amdgpu.work-item-intrinsics.deprecated.ll +++ b/test/CodeGen/AMDGPU/amdgpu.work-item-intrinsics.deprecated.ll @@ -4,21 +4,6 @@ ; Legacy intrinsics that just read implicit parameters -; FUNC-LABEL: {{^}}workdim_legacy: -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0xb -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x2c -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NOHSA: buffer_store_dword [[VVAL]] - -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV {{\*? *}}[[VAL]], KC0[2].Z -define void @workdim_legacy (i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.AMDGPU.read.workdim() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - ; FUNC-LABEL: {{^}}ngroups_x: ; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x0 ; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x0 @@ -263,6 +248,4 @@ declare i32 @llvm.r600.read.tidig.x() #0 declare i32 @llvm.r600.read.tidig.y() #0 declare i32 @llvm.r600.read.tidig.z() #0 -declare i32 @llvm.AMDGPU.read.workdim() #0 - attributes #0 = { readnone } diff --git a/test/CodeGen/AMDGPU/and.ll b/test/CodeGen/AMDGPU/and.ll index 0046bc93826e..d3d3cec9bbbf 100644 --- a/test/CodeGen/AMDGPU/and.ll +++ b/test/CodeGen/AMDGPU/and.ll @@ -258,10 +258,10 @@ define void @v_and_constant_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) } ; FUNC-LABEL: {{^}}v_and_multi_use_constant_i64: -; SI: buffer_load_dwordx2 v{{\[}}[[LO0:[0-9]+]]:[[HI0:[0-9]+]]{{\]}} -; SI: buffer_load_dwordx2 v{{\[}}[[LO1:[0-9]+]]:[[HI1:[0-9]+]]{{\]}} -; SI-DAG: s_mov_b32 [[KLO:s[0-9]+]], 0xab19b207{{$}} +; SI-DAG: buffer_load_dwordx2 v{{\[}}[[LO0:[0-9]+]]:[[HI0:[0-9]+]]{{\]}} +; SI-DAG: buffer_load_dwordx2 v{{\[}}[[LO1:[0-9]+]]:[[HI1:[0-9]+]]{{\]}} ; SI-DAG: s_movk_i32 [[KHI:s[0-9]+]], 0x11e{{$}} +; SI-DAG: s_mov_b32 [[KLO:s[0-9]+]], 0xab19b207{{$}} ; SI-DAG: v_and_b32_e32 {{v[0-9]+}}, [[KLO]], v[[LO0]] ; SI-DAG: v_and_b32_e32 {{v[0-9]+}}, [[KHI]], v[[HI0]] ; SI-DAG: v_and_b32_e32 {{v[0-9]+}}, [[KLO]], v[[LO1]] @@ -284,10 +284,9 @@ define void @v_and_multi_use_constant_i64(i64 addrspace(1)* %out, i64 addrspace( ; SI: buffer_load_dwordx2 v{{\[}}[[LO1:[0-9]+]]:[[HI1:[0-9]+]]{{\]}} ; SI-NOT: and ; SI: v_and_b32_e32 v[[RESLO0:[0-9]+]], 63, v[[LO0]] -; SI-NOT: and -; SI: buffer_store_dwordx2 v{{\[}}[[RESLO0]] ; SI: v_and_b32_e32 v[[RESLO1:[0-9]+]], 63, v[[LO1]] ; SI-NOT: and +; SI: buffer_store_dwordx2 v{{\[}}[[RESLO0]] ; SI: buffer_store_dwordx2 v{{\[}}[[RESLO1]] define void @v_and_multi_use_inline_imm_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) { %a = load volatile i64, i64 addrspace(1)* %aptr @@ -325,6 +324,20 @@ define void @v_and_inline_imm_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %apt ret void } +; FIXME: Should be able to reduce load width +; FUNC-LABEL: {{^}}v_and_inline_neg_imm_i64: +; SI: buffer_load_dwordx2 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} +; SI-NOT: and +; SI: v_and_b32_e32 v[[VAL_LO]], -8, v[[VAL_LO]] +; SI-NOT: and +; SI: buffer_store_dwordx2 v{{\[}}[[VAL_LO]]:[[VAL_HI]]{{\]}} +define void @v_and_inline_neg_imm_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr) { + %a = load i64, i64 addrspace(1)* %aptr, align 8 + %and = and i64 %a, -8 + store i64 %and, i64 addrspace(1)* %out, align 8 + ret void +} + ; FUNC-LABEL: {{^}}s_and_inline_imm_64_i64 ; SI: s_load_dword ; SI-NOT: and @@ -486,8 +499,8 @@ define void @s_and_inline_imm_neg_4.0_i64(i64 addrspace(1)* %out, i64 addrspace( ; low 32-bits, which is not a valid 64-bit inline immmediate. ; FUNC-LABEL: {{^}}s_and_inline_imm_f32_4.0_i64: -; SI: s_load_dword s ; SI: s_load_dwordx2 +; SI: s_load_dword s ; SI-NOT: and ; SI: s_and_b32 s[[K_HI:[0-9]+]], s{{[0-9]+}}, 4.0 ; SI-NOT: and diff --git a/test/CodeGen/AMDGPU/anonymous-gv.ll b/test/CodeGen/AMDGPU/anonymous-gv.ll new file mode 100644 index 000000000000..f37b0f3382f4 --- /dev/null +++ b/test/CodeGen/AMDGPU/anonymous-gv.ll @@ -0,0 +1,18 @@ +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji | FileCheck %s + +; Make sure we don't crash on a global variable with no name. +@0 = external addrspace(1) global i32 + +; CHECK-LABEL: {{^}}test: +; CHECK: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, __unnamed_1 +; CHECK: s_endpgm +define void @test() { + store i32 1, i32 addrspace(1)* @0 + ret void +} + +; CHECK-LABEL: {{^}}__unnamed_2: +; CHECK: s_endpgm +define void @1() { + ret void +} diff --git a/test/CodeGen/AMDGPU/anyext.ll b/test/CodeGen/AMDGPU/anyext.ll index 48d8f3122495..a1d3715a095c 100644 --- a/test/CodeGen/AMDGPU/anyext.ll +++ b/test/CodeGen/AMDGPU/anyext.ll @@ -1,15 +1,40 @@ -; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s -; CHECK-LABEL: {{^}}anyext_i1_i32: -; CHECK: v_cndmask_b32_e64 +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone +declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone + +; GCN-LABEL: {{^}}anyext_i1_i32: +; GCN: v_cndmask_b32_e64 define void @anyext_i1_i32(i32 addrspace(1)* %out, i32 %cond) { entry: - %0 = icmp eq i32 %cond, 0 - %1 = zext i1 %0 to i8 - %2 = xor i8 %1, -1 - %3 = and i8 %2, 1 - %4 = zext i8 %3 to i32 - store i32 %4, i32 addrspace(1)* %out + %tmp = icmp eq i32 %cond, 0 + %tmp1 = zext i1 %tmp to i8 + %tmp2 = xor i8 %tmp1, -1 + %tmp3 = and i8 %tmp2, 1 + %tmp4 = zext i8 %tmp3 to i32 + store i32 %tmp4, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}s_anyext_i16_i32: +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], +; VI: v_xor_b32_e32 [[XOR:v[0-9]+]], -1, [[ADD]] +; VI: v_and_b32_e32 [[AND:v[0-9]+]], 1, [[XOR]] +; VI: buffer_store_dword [[AND]] +define void @s_anyext_i16_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %a, i16 addrspace(1)* %b) { +entry: + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %tid.y = call i32 @llvm.amdgcn.workitem.id.y() + %a.ptr = getelementptr i16, i16 addrspace(1)* %a, i32 %tid.x + %b.ptr = getelementptr i16, i16 addrspace(1)* %b, i32 %tid.y + %a.l = load i16, i16 addrspace(1)* %a.ptr + %b.l = load i16, i16 addrspace(1)* %b.ptr + %tmp = add i16 %a.l, %b.l + %tmp1 = trunc i16 %tmp to i8 + %tmp2 = xor i8 %tmp1, -1 + %tmp3 = and i8 %tmp2, 1 + %tmp4 = zext i8 %tmp3 to i32 + store i32 %tmp4, i32 addrspace(1)* %out ret void } diff --git a/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll b/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll index b00fff0a6f9a..f190bd0cb01e 100644 --- a/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll +++ b/test/CodeGen/AMDGPU/array-ptr-calc-i32.ll @@ -47,6 +47,6 @@ define void @test_private_array_ptr_calc(i32 addrspace(1)* noalias %out, i32 add ret void } -attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="1" } +attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" } attributes #1 = { nounwind readnone } attributes #2 = { nounwind convergent } diff --git a/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll b/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll new file mode 100644 index 000000000000..cab377feacb2 --- /dev/null +++ b/test/CodeGen/AMDGPU/attr-amdgpu-flat-work-group-size.ll @@ -0,0 +1,129 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}min_64_max_64: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @min_64_max_64() #0 { +entry: + ret void +} +attributes #0 = {"amdgpu-flat-work-group-size"="64,64"} + +; CHECK-LABEL: {{^}}min_64_max_128: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @min_64_max_128() #1 { +entry: + ret void +} +attributes #1 = {"amdgpu-flat-work-group-size"="64,128"} + +; CHECK-LABEL: {{^}}min_128_max_128: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @min_128_max_128() #2 { +entry: + ret void +} +attributes #2 = {"amdgpu-flat-work-group-size"="128,128"} + +; CHECK-LABEL: {{^}}min_1024_max_2048 +; CHECK: SGPRBlocks: 1 +; CHECK: VGPRBlocks: 7 +; CHECK: NumSGPRsForWavesPerEU: 13 +; CHECK: NumVGPRsForWavesPerEU: 32 +@var = addrspace(1) global float 0.0 +define void @min_1024_max_2048() #3 { + %val0 = load volatile float, float addrspace(1)* @var + %val1 = load volatile float, float addrspace(1)* @var + %val2 = load volatile float, float addrspace(1)* @var + %val3 = load volatile float, float addrspace(1)* @var + %val4 = load volatile float, float addrspace(1)* @var + %val5 = load volatile float, float addrspace(1)* @var + %val6 = load volatile float, float addrspace(1)* @var + %val7 = load volatile float, float addrspace(1)* @var + %val8 = load volatile float, float addrspace(1)* @var + %val9 = load volatile float, float addrspace(1)* @var + %val10 = load volatile float, float addrspace(1)* @var + %val11 = load volatile float, float addrspace(1)* @var + %val12 = load volatile float, float addrspace(1)* @var + %val13 = load volatile float, float addrspace(1)* @var + %val14 = load volatile float, float addrspace(1)* @var + %val15 = load volatile float, float addrspace(1)* @var + %val16 = load volatile float, float addrspace(1)* @var + %val17 = load volatile float, float addrspace(1)* @var + %val18 = load volatile float, float addrspace(1)* @var + %val19 = load volatile float, float addrspace(1)* @var + %val20 = load volatile float, float addrspace(1)* @var + %val21 = load volatile float, float addrspace(1)* @var + %val22 = load volatile float, float addrspace(1)* @var + %val23 = load volatile float, float addrspace(1)* @var + %val24 = load volatile float, float addrspace(1)* @var + %val25 = load volatile float, float addrspace(1)* @var + %val26 = load volatile float, float addrspace(1)* @var + %val27 = load volatile float, float addrspace(1)* @var + %val28 = load volatile float, float addrspace(1)* @var + %val29 = load volatile float, float addrspace(1)* @var + %val30 = load volatile float, float addrspace(1)* @var + %val31 = load volatile float, float addrspace(1)* @var + %val32 = load volatile float, float addrspace(1)* @var + %val33 = load volatile float, float addrspace(1)* @var + %val34 = load volatile float, float addrspace(1)* @var + %val35 = load volatile float, float addrspace(1)* @var + %val36 = load volatile float, float addrspace(1)* @var + %val37 = load volatile float, float addrspace(1)* @var + %val38 = load volatile float, float addrspace(1)* @var + %val39 = load volatile float, float addrspace(1)* @var + %val40 = load volatile float, float addrspace(1)* @var + + store volatile float %val0, float addrspace(1)* @var + store volatile float %val1, float addrspace(1)* @var + store volatile float %val2, float addrspace(1)* @var + store volatile float %val3, float addrspace(1)* @var + store volatile float %val4, float addrspace(1)* @var + store volatile float %val5, float addrspace(1)* @var + store volatile float %val6, float addrspace(1)* @var + store volatile float %val7, float addrspace(1)* @var + store volatile float %val8, float addrspace(1)* @var + store volatile float %val9, float addrspace(1)* @var + store volatile float %val10, float addrspace(1)* @var + store volatile float %val11, float addrspace(1)* @var + store volatile float %val12, float addrspace(1)* @var + store volatile float %val13, float addrspace(1)* @var + store volatile float %val14, float addrspace(1)* @var + store volatile float %val15, float addrspace(1)* @var + store volatile float %val16, float addrspace(1)* @var + store volatile float %val17, float addrspace(1)* @var + store volatile float %val18, float addrspace(1)* @var + store volatile float %val19, float addrspace(1)* @var + store volatile float %val20, float addrspace(1)* @var + store volatile float %val21, float addrspace(1)* @var + store volatile float %val22, float addrspace(1)* @var + store volatile float %val23, float addrspace(1)* @var + store volatile float %val24, float addrspace(1)* @var + store volatile float %val25, float addrspace(1)* @var + store volatile float %val26, float addrspace(1)* @var + store volatile float %val27, float addrspace(1)* @var + store volatile float %val28, float addrspace(1)* @var + store volatile float %val29, float addrspace(1)* @var + store volatile float %val30, float addrspace(1)* @var + store volatile float %val31, float addrspace(1)* @var + store volatile float %val32, float addrspace(1)* @var + store volatile float %val33, float addrspace(1)* @var + store volatile float %val34, float addrspace(1)* @var + store volatile float %val35, float addrspace(1)* @var + store volatile float %val36, float addrspace(1)* @var + store volatile float %val37, float addrspace(1)* @var + store volatile float %val38, float addrspace(1)* @var + store volatile float %val39, float addrspace(1)* @var + store volatile float %val40, float addrspace(1)* @var + + ret void +} +attributes #3 = {"amdgpu-flat-work-group-size"="1024,2048"} diff --git a/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll b/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll new file mode 100644 index 000000000000..e4f6e72e6977 --- /dev/null +++ b/test/CodeGen/AMDGPU/attr-amdgpu-num-sgpr.ll @@ -0,0 +1,127 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=TOSGPR -check-prefix=ALL %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -amdgpu-spill-sgpr-to-smem=1 -verify-machineinstrs < %s | FileCheck -check-prefix=TOSMEM -check-prefix=ALL %s + +; If spilling to smem, additional registers are used for the resource +; descriptor. + +; ALL-LABEL: {{^}}max_12_sgprs: + +; FIXME: Should be ablo to skip this copying of the private segment +; buffer because all the SGPR spills are to VGPRs. + +; ALL: s_mov_b64 s[10:11], s[2:3] +; ALL: s_mov_b64 s[8:9], s[0:1] +; ALL: SGPRBlocks: 1 +; ALL: NumSGPRsForWavesPerEU: 14 +define void @max_12_sgprs(i32 addrspace(1)* %out1, + + i32 addrspace(1)* %out2, + i32 addrspace(1)* %out3, + i32 addrspace(1)* %out4, + i32 %one, i32 %two, i32 %three, i32 %four) #0 { + store i32 %one, i32 addrspace(1)* %out1 + store i32 %two, i32 addrspace(1)* %out2 + store i32 %three, i32 addrspace(1)* %out3 + store i32 %four, i32 addrspace(1)* %out4 + ret void +} + +; private resource: 4 +; scratch wave offset: 1 +; workgroup ids: 3 +; dispatch id: 2 +; queue ptr: 2 +; flat scratch init: 2 +; --------------------- +; total: 14 + +; + reserved vcc = 16 + +; Because we can't handle re-using the last few input registers as the +; special vcc etc. registers (as well as decide to not use the unused +; features when the number of registers is frozen), this ends up using +; more than expected. + +; ALL-LABEL: {{^}}max_12_sgprs_14_input_sgprs: +; TOSGPR: SGPRBlocks: 1 +; TOSGPR: NumSGPRsForWavesPerEU: 16 + +; TOSMEM: s_mov_b64 s[10:11], s[2:3] +; TOSMEM: s_mov_b64 s[8:9], s[0:1] +; TOSMEM: s_mov_b32 s7, s13 + +; TOSMEM: SGPRBlocks: 1 +; TOSMEM: NumSGPRsForWavesPerEU: 16 +define void @max_12_sgprs_14_input_sgprs(i32 addrspace(1)* %out1, + i32 addrspace(1)* %out2, + i32 addrspace(1)* %out3, + i32 addrspace(1)* %out4, + i32 %one, i32 %two, i32 %three, i32 %four) #2 { + store volatile i32 0, i32* undef + %x.0 = call i32 @llvm.amdgcn.workgroup.id.x() + store volatile i32 %x.0, i32 addrspace(1)* undef + %x.1 = call i32 @llvm.amdgcn.workgroup.id.y() + store volatile i32 %x.0, i32 addrspace(1)* undef + %x.2 = call i32 @llvm.amdgcn.workgroup.id.z() + store volatile i32 %x.0, i32 addrspace(1)* undef + %x.3 = call i64 @llvm.amdgcn.dispatch.id() + store volatile i64 %x.3, i64 addrspace(1)* undef + %x.4 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() + store volatile i8 addrspace(2)* %x.4, i8 addrspace(2)* addrspace(1)* undef + %x.5 = call i8 addrspace(2)* @llvm.amdgcn.queue.ptr() + store volatile i8 addrspace(2)* %x.5, i8 addrspace(2)* addrspace(1)* undef + + store i32 %one, i32 addrspace(1)* %out1 + store i32 %two, i32 addrspace(1)* %out2 + store i32 %three, i32 addrspace(1)* %out3 + store i32 %four, i32 addrspace(1)* %out4 + ret void +} + +; The following test is commented out for now; http://llvm.org/PR31230 +; XALL-LABEL: max_12_sgprs_12_input_sgprs{{$}} +; ; Make sure copies for input buffer are not clobbered. This requires +; ; swapping the order the registers are copied from what normally +; ; happens. + +; XTOSMEM: s_mov_b32 s5, s11 +; XTOSMEM: s_add_u32 m0, s5, +; XTOSMEM: s_buffer_store_dword vcc_lo, s[0:3], m0 + +; XALL: SGPRBlocks: 2 +; XALL: NumSGPRsForWavesPerEU: 18 +;define void @max_12_sgprs_12_input_sgprs(i32 addrspace(1)* %out1, +; i32 addrspace(1)* %out2, +; i32 addrspace(1)* %out3, +; i32 addrspace(1)* %out4, +; i32 %one, i32 %two, i32 %three, i32 %four) #2 { +; store volatile i32 0, i32* undef +; %x.0 = call i32 @llvm.amdgcn.workgroup.id.x() +; store volatile i32 %x.0, i32 addrspace(1)* undef +; %x.1 = call i32 @llvm.amdgcn.workgroup.id.y() +; store volatile i32 %x.0, i32 addrspace(1)* undef +; %x.2 = call i32 @llvm.amdgcn.workgroup.id.z() +; store volatile i32 %x.0, i32 addrspace(1)* undef +; %x.3 = call i64 @llvm.amdgcn.dispatch.id() +; store volatile i64 %x.3, i64 addrspace(1)* undef +; %x.4 = call i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() +; store volatile i8 addrspace(2)* %x.4, i8 addrspace(2)* addrspace(1)* undef +; +; store i32 %one, i32 addrspace(1)* %out1 +; store i32 %two, i32 addrspace(1)* %out2 +; store i32 %three, i32 addrspace(1)* %out3 +; store i32 %four, i32 addrspace(1)* %out4 +; ret void +;} + +declare i32 @llvm.amdgcn.workgroup.id.x() #1 +declare i32 @llvm.amdgcn.workgroup.id.y() #1 +declare i32 @llvm.amdgcn.workgroup.id.z() #1 +declare i64 @llvm.amdgcn.dispatch.id() #1 +declare i8 addrspace(2)* @llvm.amdgcn.dispatch.ptr() #1 +declare i8 addrspace(2)* @llvm.amdgcn.queue.ptr() #1 + +attributes #0 = { nounwind "amdgpu-num-sgpr"="14" } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind "amdgpu-num-sgpr"="12" } +attributes #3 = { nounwind "amdgpu-num-sgpr"="11" } diff --git a/test/CodeGen/AMDGPU/attr-amdgpu-num-vgpr.ll b/test/CodeGen/AMDGPU/attr-amdgpu-num-vgpr.ll new file mode 100644 index 000000000000..97feb7276b7d --- /dev/null +++ b/test/CodeGen/AMDGPU/attr-amdgpu-num-vgpr.ll @@ -0,0 +1,75 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s + +@var = addrspace(1) global float 0.0 + +; CHECK-LABEL: {{^}}max_20_vgprs: +; CHECK: VGPRBlocks: 4 +; CHECK: NumVGPRsForWavesPerEU: 20 +define void @max_20_vgprs() #1 { + %val0 = load volatile float, float addrspace(1)* @var + %val1 = load volatile float, float addrspace(1)* @var + %val2 = load volatile float, float addrspace(1)* @var + %val3 = load volatile float, float addrspace(1)* @var + %val4 = load volatile float, float addrspace(1)* @var + %val5 = load volatile float, float addrspace(1)* @var + %val6 = load volatile float, float addrspace(1)* @var + %val7 = load volatile float, float addrspace(1)* @var + %val8 = load volatile float, float addrspace(1)* @var + %val9 = load volatile float, float addrspace(1)* @var + %val10 = load volatile float, float addrspace(1)* @var + %val11 = load volatile float, float addrspace(1)* @var + %val12 = load volatile float, float addrspace(1)* @var + %val13 = load volatile float, float addrspace(1)* @var + %val14 = load volatile float, float addrspace(1)* @var + %val15 = load volatile float, float addrspace(1)* @var + %val16 = load volatile float, float addrspace(1)* @var + %val17 = load volatile float, float addrspace(1)* @var + %val18 = load volatile float, float addrspace(1)* @var + %val19 = load volatile float, float addrspace(1)* @var + %val20 = load volatile float, float addrspace(1)* @var + %val21 = load volatile float, float addrspace(1)* @var + %val22 = load volatile float, float addrspace(1)* @var + %val23 = load volatile float, float addrspace(1)* @var + %val24 = load volatile float, float addrspace(1)* @var + %val25 = load volatile float, float addrspace(1)* @var + %val26 = load volatile float, float addrspace(1)* @var + %val27 = load volatile float, float addrspace(1)* @var + %val28 = load volatile float, float addrspace(1)* @var + %val29 = load volatile float, float addrspace(1)* @var + %val30 = load volatile float, float addrspace(1)* @var + + store volatile float %val0, float addrspace(1)* @var + store volatile float %val1, float addrspace(1)* @var + store volatile float %val2, float addrspace(1)* @var + store volatile float %val3, float addrspace(1)* @var + store volatile float %val4, float addrspace(1)* @var + store volatile float %val5, float addrspace(1)* @var + store volatile float %val6, float addrspace(1)* @var + store volatile float %val7, float addrspace(1)* @var + store volatile float %val8, float addrspace(1)* @var + store volatile float %val9, float addrspace(1)* @var + store volatile float %val10, float addrspace(1)* @var + store volatile float %val11, float addrspace(1)* @var + store volatile float %val12, float addrspace(1)* @var + store volatile float %val13, float addrspace(1)* @var + store volatile float %val14, float addrspace(1)* @var + store volatile float %val15, float addrspace(1)* @var + store volatile float %val16, float addrspace(1)* @var + store volatile float %val17, float addrspace(1)* @var + store volatile float %val18, float addrspace(1)* @var + store volatile float %val19, float addrspace(1)* @var + store volatile float %val20, float addrspace(1)* @var + store volatile float %val21, float addrspace(1)* @var + store volatile float %val22, float addrspace(1)* @var + store volatile float %val23, float addrspace(1)* @var + store volatile float %val24, float addrspace(1)* @var + store volatile float %val25, float addrspace(1)* @var + store volatile float %val26, float addrspace(1)* @var + store volatile float %val27, float addrspace(1)* @var + store volatile float %val28, float addrspace(1)* @var + store volatile float %val29, float addrspace(1)* @var + store volatile float %val30, float addrspace(1)* @var + + ret void +} +attributes #1 = {"amdgpu-num-vgpr"="20"} diff --git a/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll b/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll new file mode 100644 index 000000000000..4f4efccc2260 --- /dev/null +++ b/test/CodeGen/AMDGPU/attr-amdgpu-waves-per-eu.ll @@ -0,0 +1,190 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s + +; Exactly 1 wave per execution unit. +; CHECK-LABEL: {{^}}empty_exactly_1: +; CHECK: SGPRBlocks: 12 +; CHECK: VGPRBlocks: 32 +; CHECK: NumSGPRsForWavesPerEU: 97 +; CHECK: NumVGPRsForWavesPerEU: 129 +define void @empty_exactly_1() #0 { +entry: + ret void +} +attributes #0 = {"amdgpu-waves-per-eu"="1,1"} + +; Exactly 5 waves per execution unit. +; CHECK-LABEL: {{^}}empty_exactly_5: +; CHECK: SGPRBlocks: 12 +; CHECK: VGPRBlocks: 10 +; CHECK: NumSGPRsForWavesPerEU: 97 +; CHECK: NumVGPRsForWavesPerEU: 41 +define void @empty_exactly_5() #1 { +entry: + ret void +} +attributes #1 = {"amdgpu-waves-per-eu"="5,5"} + +; Exactly 10 waves per execution unit. +; CHECK-LABEL: {{^}}empty_exactly_10: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_exactly_10() #2 { +entry: + ret void +} +attributes #2 = {"amdgpu-waves-per-eu"="10,10"} + +; At least 1 wave per execution unit. +; CHECK-LABEL: {{^}}empty_at_least_1: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_at_least_1() #3 { +entry: + ret void +} +attributes #3 = {"amdgpu-waves-per-eu"="1"} + +; At least 5 waves per execution unit. +; CHECK-LABEL: {{^}}empty_at_least_5: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_at_least_5() #4 { +entry: + ret void +} +attributes #4 = {"amdgpu-waves-per-eu"="5"} + +; At least 10 waves per execution unit. +; CHECK-LABEL: {{^}}empty_at_least_10: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_at_least_10() #5 { +entry: + ret void +} +attributes #5 = {"amdgpu-waves-per-eu"="10"} + +; At most 1 wave per execution unit (same as @empty_exactly_1). + +; At most 5 waves per execution unit. +; CHECK-LABEL: {{^}}empty_at_most_5: +; CHECK: SGPRBlocks: 12 +; CHECK: VGPRBlocks: 10 +; CHECK: NumSGPRsForWavesPerEU: 97 +; CHECK: NumVGPRsForWavesPerEU: 41 +define void @empty_at_most_5() #6 { +entry: + ret void +} +attributes #6 = {"amdgpu-waves-per-eu"="1,5"} + +; At most 10 waves per execution unit. +; CHECK-LABEL: {{^}}empty_at_most_10: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_at_most_10() #7 { +entry: + ret void +} +attributes #7 = {"amdgpu-waves-per-eu"="1,10"} + +; Between 1 and 5 waves per execution unit (same as @empty_at_most_5). + +; Between 5 and 10 waves per execution unit. +; CHECK-LABEL: {{^}}empty_between_5_and_10: +; CHECK: SGPRBlocks: 0 +; CHECK: VGPRBlocks: 0 +; CHECK: NumSGPRsForWavesPerEU: 1 +; CHECK: NumVGPRsForWavesPerEU: 1 +define void @empty_between_5_and_10() #8 { +entry: + ret void +} +attributes #8 = {"amdgpu-waves-per-eu"="5,10"} + +@var = addrspace(1) global float 0.0 + +; Exactly 10 waves per execution unit. +; CHECK-LABEL: {{^}}exactly_10: +; CHECK: SGPRBlocks: 1 +; CHECK: VGPRBlocks: 5 +; CHECK: NumSGPRsForWavesPerEU: 13 +; CHECK: NumVGPRsForWavesPerEU: 24 +define void @exactly_10() #9 { + %val0 = load volatile float, float addrspace(1)* @var + %val1 = load volatile float, float addrspace(1)* @var + %val2 = load volatile float, float addrspace(1)* @var + %val3 = load volatile float, float addrspace(1)* @var + %val4 = load volatile float, float addrspace(1)* @var + %val5 = load volatile float, float addrspace(1)* @var + %val6 = load volatile float, float addrspace(1)* @var + %val7 = load volatile float, float addrspace(1)* @var + %val8 = load volatile float, float addrspace(1)* @var + %val9 = load volatile float, float addrspace(1)* @var + %val10 = load volatile float, float addrspace(1)* @var + %val11 = load volatile float, float addrspace(1)* @var + %val12 = load volatile float, float addrspace(1)* @var + %val13 = load volatile float, float addrspace(1)* @var + %val14 = load volatile float, float addrspace(1)* @var + %val15 = load volatile float, float addrspace(1)* @var + %val16 = load volatile float, float addrspace(1)* @var + %val17 = load volatile float, float addrspace(1)* @var + %val18 = load volatile float, float addrspace(1)* @var + %val19 = load volatile float, float addrspace(1)* @var + %val20 = load volatile float, float addrspace(1)* @var + %val21 = load volatile float, float addrspace(1)* @var + %val22 = load volatile float, float addrspace(1)* @var + %val23 = load volatile float, float addrspace(1)* @var + %val24 = load volatile float, float addrspace(1)* @var + %val25 = load volatile float, float addrspace(1)* @var + %val26 = load volatile float, float addrspace(1)* @var + %val27 = load volatile float, float addrspace(1)* @var + %val28 = load volatile float, float addrspace(1)* @var + %val29 = load volatile float, float addrspace(1)* @var + %val30 = load volatile float, float addrspace(1)* @var + + store volatile float %val0, float addrspace(1)* @var + store volatile float %val1, float addrspace(1)* @var + store volatile float %val2, float addrspace(1)* @var + store volatile float %val3, float addrspace(1)* @var + store volatile float %val4, float addrspace(1)* @var + store volatile float %val5, float addrspace(1)* @var + store volatile float %val6, float addrspace(1)* @var + store volatile float %val7, float addrspace(1)* @var + store volatile float %val8, float addrspace(1)* @var + store volatile float %val9, float addrspace(1)* @var + store volatile float %val10, float addrspace(1)* @var + store volatile float %val11, float addrspace(1)* @var + store volatile float %val12, float addrspace(1)* @var + store volatile float %val13, float addrspace(1)* @var + store volatile float %val14, float addrspace(1)* @var + store volatile float %val15, float addrspace(1)* @var + store volatile float %val16, float addrspace(1)* @var + store volatile float %val17, float addrspace(1)* @var + store volatile float %val18, float addrspace(1)* @var + store volatile float %val19, float addrspace(1)* @var + store volatile float %val20, float addrspace(1)* @var + store volatile float %val21, float addrspace(1)* @var + store volatile float %val22, float addrspace(1)* @var + store volatile float %val23, float addrspace(1)* @var + store volatile float %val24, float addrspace(1)* @var + store volatile float %val25, float addrspace(1)* @var + store volatile float %val26, float addrspace(1)* @var + store volatile float %val27, float addrspace(1)* @var + store volatile float %val28, float addrspace(1)* @var + store volatile float %val29, float addrspace(1)* @var + store volatile float %val30, float addrspace(1)* @var + + ret void +} +attributes #9 = {"amdgpu-waves-per-eu"="10,10"} diff --git a/test/CodeGen/AMDGPU/attr-unparseable.ll b/test/CodeGen/AMDGPU/attr-unparseable.ll new file mode 100644 index 000000000000..0282bc34c0ee --- /dev/null +++ b/test/CodeGen/AMDGPU/attr-unparseable.ll @@ -0,0 +1,57 @@ +; RUN: not llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s 2>&1 | FileCheck %s + +; CHECK: can't parse integer attribute amdgpu-num-sgpr +define void @unparseable_single_0() #0 { +entry: + ret void +} +attributes #0 = {"amdgpu-num-sgpr"} + +; CHECK: can't parse integer attribute amdgpu-num-sgpr +define void @unparseable_single_1() #1 { +entry: + ret void +} +attributes #1 = {"amdgpu-num-sgpr"="k"} + +; CHECK: can't parse integer attribute amdgpu-num-sgpr +define void @unparseable_single_2() #2 { +entry: + ret void +} +attributes #2 = {"amdgpu-num-sgpr"="1,2"} + +; CHECK: can't parse first integer attribute amdgpu-flat-work-group-size +define void @unparseable_pair_0() #3 { +entry: + ret void +} +attributes #3 = {"amdgpu-flat-work-group-size"} + +; CHECK: can't parse first integer attribute amdgpu-flat-work-group-size +define void @unparseable_pair_1() #4 { +entry: + ret void +} +attributes #4 = {"amdgpu-flat-work-group-size"="k"} + +; CHECK: can't parse second integer attribute amdgpu-flat-work-group-size +define void @unparseable_pair_2() #5 { +entry: + ret void +} +attributes #5 = {"amdgpu-flat-work-group-size"="1"} + +; CHECK: can't parse second integer attribute amdgpu-flat-work-group-size +define void @unparseable_pair_3() #6 { +entry: + ret void +} +attributes #6 = {"amdgpu-flat-work-group-size"="1,k"} + +; CHECK: can't parse second integer attribute amdgpu-flat-work-group-size +define void @unparseable_pair_4() #7 { +entry: + ret void +} +attributes #7 = {"amdgpu-flat-work-group-size"="1,2,3"} diff --git a/test/CodeGen/AMDGPU/basic-branch.ll b/test/CodeGen/AMDGPU/basic-branch.ll index 00636240bc6c..24874ee7fa98 100644 --- a/test/CodeGen/AMDGPU/basic-branch.ll +++ b/test/CodeGen/AMDGPU/basic-branch.ll @@ -1,5 +1,5 @@ ; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCNNOOPT -check-prefix=GCN %s -; RUN: llc -O0 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCNNOOPT -check-prefix=GCN %s +; RUN: llc -O0 -march=amdgcn -mcpu=tonga -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCNNOOPT -check-prefix=GCN %s ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCNOPT -check-prefix=GCN %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCNOPT -check-prefix=GCN %s @@ -8,13 +8,15 @@ ; GCNNOOPT: v_writelane_b32 ; GCN: s_cbranch_scc1 [[END:BB[0-9]+_[0-9]+]] + ; GCN: ; BB#1 ; GCNNOOPT: v_readlane_b32 ; GCNNOOPT: v_readlane_b32 ; GCN: buffer_store_dword -; GCN: s_endpgm +; GCNOPT-NEXT: s_waitcnt vmcnt(0) expcnt(0) +; TODO: This waitcnt can be eliminated -; GCN: {{^}}[[END]] +; GCN: {{^}}[[END]]: ; GCN: s_endpgm define void @test_branch(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i32 %val) #0 { %cmp = icmp ne i32 %val, 0 @@ -31,13 +33,14 @@ end: ; GCN-LABEL: {{^}}test_brcc_i1: ; GCN: buffer_load_ubyte ; GCN: v_and_b32_e32 v{{[0-9]+}}, 1, -; GCN: v_cmp_eq_i32_e32 vcc, +; GCN: v_cmp_eq_u32_e32 vcc, ; GCN: s_cbranch_vccnz [[END:BB[0-9]+_[0-9]+]] ; GCN: buffer_store_dword -; GCN: s_endpgm +; GCNOPT-NEXT: s_waitcnt vmcnt(0) expcnt(0) +; TODO: This waitcnt can be eliminated -; GCN: {{^}}[[END]] +; GCN: {{^}}[[END]]: ; GCN: s_endpgm define void @test_brcc_i1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in, i1 %val) #0 { %cmp0 = icmp ne i1 %val, 0 diff --git a/test/CodeGen/AMDGPU/bitcast-vector-extract.ll b/test/CodeGen/AMDGPU/bitcast-vector-extract.ll new file mode 100644 index 000000000000..2482fa761b19 --- /dev/null +++ b/test/CodeGen/AMDGPU/bitcast-vector-extract.ll @@ -0,0 +1,69 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; The bitcast should be pushed through the bitcasts so the vectors can +; be broken down and the shared components can be CSEd + +; GCN-LABEL: {{^}}store_bitcast_constant_v8i32_to_v8f32: +; GCN: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +define void @store_bitcast_constant_v8i32_to_v8f32(<8 x float> addrspace(1)* %out, <8 x i32> %vec) { + %vec0.bc = bitcast <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 8> to <8 x float> + store volatile <8 x float> %vec0.bc, <8 x float> addrspace(1)* %out + + %vec1.bc = bitcast <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 9> to <8 x float> + store volatile <8 x float> %vec1.bc, <8 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_bitcast_constant_v4i64_to_v8f32: +; GCN: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +define void @store_bitcast_constant_v4i64_to_v8f32(<8 x float> addrspace(1)* %out, <4 x i64> %vec) { + %vec0.bc = bitcast <4 x i64> <i64 7, i64 7, i64 7, i64 8> to <8 x float> + store volatile <8 x float> %vec0.bc, <8 x float> addrspace(1)* %out + + %vec1.bc = bitcast <4 x i64> <i64 7, i64 7, i64 7, i64 9> to <8 x float> + store volatile <8 x float> %vec1.bc, <8 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_bitcast_constant_v4i64_to_v4f64: +; GCN: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +define void @store_bitcast_constant_v4i64_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i64> %vec) { + %vec0.bc = bitcast <4 x i64> <i64 7, i64 7, i64 7, i64 8> to <4 x double> + store volatile <4 x double> %vec0.bc, <4 x double> addrspace(1)* %out + + %vec1.bc = bitcast <4 x i64> <i64 7, i64 7, i64 7, i64 9> to <4 x double> + store volatile <4 x double> %vec1.bc, <4 x double> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_bitcast_constant_v8i32_to_v16i16: +; GCN: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +; GCN-NOT: v_mov_b32 +; GCN: buffer_store_dwordx4 +define void @store_bitcast_constant_v8i32_to_v16i16(<8 x float> addrspace(1)* %out, <16 x i16> %vec) { + %vec0.bc = bitcast <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 8> to <8 x float> + store volatile <8 x float> %vec0.bc, <8 x float> addrspace(1)* %out + + %vec1.bc = bitcast <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 9> to <8 x float> + store volatile <8 x float> %vec1.bc, <8 x float> addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll b/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll index 150e3430a5e9..f7dc1a9d37e8 100644 --- a/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll +++ b/test/CodeGen/AMDGPU/bitreverse-inline-immediates.ll @@ -156,3 +156,66 @@ define void @materialize_rev_1.0_i64(i64 addrspace(1)* %out) { store i64 508, i64 addrspace(1)* %out ret void } + +; GCN-LABEL: {{^}}s_materialize_0_i32: +; GCN: s_mov_b32 s{{[0-9]+}}, 0{{$}} +define void @s_materialize_0_i32() { + call void asm sideeffect "; use $0", "s"(i32 0) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_1_i32: +; GCN: s_mov_b32 s{{[0-9]+}}, 1{{$}} +define void @s_materialize_1_i32() { + call void asm sideeffect "; use $0", "s"(i32 1) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_neg1_i32: +; GCN: s_mov_b32 s{{[0-9]+}}, -1{{$}} +define void @s_materialize_neg1_i32() { + call void asm sideeffect "; use $0", "s"(i32 -1) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_signbit_i32: +; GCN: s_brev_b32 s{{[0-9]+}}, 1{{$}} +define void @s_materialize_signbit_i32() { + call void asm sideeffect "; use $0", "s"(i32 -2147483648) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_rev_64_i32: +; GCN: s_brev_b32 s{{[0-9]+}}, 64{{$}} +define void @s_materialize_rev_64_i32() { + call void asm sideeffect "; use $0", "s"(i32 33554432) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_rev_65_i32: +; GCN: s_mov_b32 s{{[0-9]+}}, 0x82000000{{$}} +define void @s_materialize_rev_65_i32() { + call void asm sideeffect "; use $0", "s"(i32 -2113929216) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_rev_neg16_i32: +; GCN: s_brev_b32 s{{[0-9]+}}, -16{{$}} +define void @s_materialize_rev_neg16_i32() { + call void asm sideeffect "; use $0", "s"(i32 268435455) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_rev_neg17_i32: +; GCN: s_mov_b32 s{{[0-9]+}}, 0xf7ffffff{{$}} +define void @s_materialize_rev_neg17_i32() { + call void asm sideeffect "; use $0", "s"(i32 -134217729) + ret void +} + +; GCN-LABEL: {{^}}s_materialize_rev_1.0_i32: +; GCN: s_movk_i32 s{{[0-9]+}}, 0x1fc{{$}} +define void @s_materialize_rev_1.0_i32() { + call void asm sideeffect "; use $0", "s"(i32 508) + ret void +} diff --git a/test/CodeGen/AMDGPU/bitreverse.ll b/test/CodeGen/AMDGPU/bitreverse.ll index 62e7904f4382..aca88a9ef2a3 100644 --- a/test/CodeGen/AMDGPU/bitreverse.ll +++ b/test/CodeGen/AMDGPU/bitreverse.ll @@ -1,5 +1,6 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC %s declare i16 @llvm.bitreverse.i16(i16) #1 declare i32 @llvm.bitreverse.i32(i32) #1 @@ -12,7 +13,7 @@ declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>) #1 declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>) #1 ; FUNC-LABEL: {{^}}s_brev_i16: -; SI: s_brev_b32 +; SI: s_brev_b32 define void @s_brev_i16(i16 addrspace(1)* noalias %out, i16 %val) #0 { %brev = call i16 @llvm.bitreverse.i16(i16 %val) #1 store i16 %brev, i16 addrspace(1)* %out @@ -79,6 +80,7 @@ define void @s_brev_i64(i64 addrspace(1)* noalias %out, i64 %val) #0 { } ; FUNC-LABEL: {{^}}v_brev_i64: +; SI-NOT: v_or_b32_e64 v{{[0-9]+}}, 0, 0 define void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %valptr) #0 { %val = load i64, i64 addrspace(1)* %valptr %brev = call i64 @llvm.bitreverse.i64(i64 %val) #1 diff --git a/test/CodeGen/AMDGPU/br_cc.f16.ll b/test/CodeGen/AMDGPU/br_cc.f16.ll new file mode 100644 index 000000000000..340d30b898e0 --- /dev/null +++ b/test/CodeGen/AMDGPU/br_cc.f16.ll @@ -0,0 +1,114 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}br_cc_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] + +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_nlt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_nlt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: s_cbranch_vccnz + +; GCN: one{{$}} +; SI: v_cvt_f16_f32_e32 v[[A_F16:[0-9]+]], v[[A_F32]] +; SI: s_branch +; VI: buffer_store_short +; VI: s_endpgm + +; GCN: two{{$}} +; SI: v_cvt_f16_f32_e32 v[[B_F16:[0-9]+]], v[[B_F32]] +; GCN: buffer_store_short v[[B_F16]] +; GCN: s_endpgm +define void @br_cc_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %fcmp = fcmp olt half %a.val, %b.val + br i1 %fcmp, label %one, label %two + +one: + store half %a.val, half addrspace(1)* %r + ret void + +two: + store half %b.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}br_cc_f16_imm_a +; SI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] + +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_ngt_f32_e32 vcc, v[[B_F32]], v[[A_F32]] +; SI: s_cbranch_vccz + +; VI: v_cmp_nlt_f16_e32 vcc, 0.5, v[[B_F16]] +; VI: s_cbranch_vccnz + +; VI: one{{$}} +; VI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x380{{0|1}}{{$}} + +; GCN: two{{$}} +; SI: v_cvt_f16_f32_e32 v[[B_F16:[0-9]+]], v[[B_F32]] + +; SI: one{{$}} +; SI: buffer_store_short v[[A_F16]] +; SI: s_endpgm + +define void @br_cc_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %fcmp = fcmp olt half 0xH3800, %b.val + br i1 %fcmp, label %one, label %two + +one: + store half 0xH3800, half addrspace(1)* %r + ret void + +two: + store half %b.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}br_cc_f16_imm_b +; SI: v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] + +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cmp_nlt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_ngt_f16_e32 vcc, 0.5, v[[A_F16]] +; GCN: s_cbranch_vccnz + +; GCN: one{{$}} +; SI: v_cvt_f16_f32_e32 v[[A_F16:[0-9]+]], v[[A_F32]] + +; GCN: two{{$}} +; VI: v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x3800{{$}} +; GCN: buffer_store_short v[[B_F16]] +; GCN: s_endpgm +define void @br_cc_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %fcmp = fcmp olt half %a.val, 0xH3800 + br i1 %fcmp, label %one, label %two + +one: + store half %a.val, half addrspace(1)* %r + ret void + +two: + store half 0xH3800, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/branch-condition-and.ll b/test/CodeGen/AMDGPU/branch-condition-and.ll new file mode 100644 index 000000000000..86d96435a29e --- /dev/null +++ b/test/CodeGen/AMDGPU/branch-condition-and.ll @@ -0,0 +1,39 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; This used to crash because during intermediate control flow lowering, there +; was a sequence +; s_mov_b64 s[0:1], exec +; s_and_b64 s[2:3], s[0:1], s[2:3] ; def & use of the same register pair +; ... +; s_mov_b64_term exec, s[2:3] +; that was not treated correctly. +; +; GCN-LABEL: {{^}}ham: +; GCN-DAG: v_cmp_lt_f32_e64 [[OTHERCC:s\[[0-9]+:[0-9]+\]]], +; GCN-DAG: v_cmp_lt_f32_e32 vcc, +; GCN: s_and_b64 [[AND:s\[[0-9]+:[0-9]+\]]], vcc, [[OTHERCC]] +; GCN: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[AND]] +; GCN: s_xor_b64 {{s\[[0-9]+:[0-9]+\]}}, exec, [[SAVED]] +; +; TODO: The following sequence is a bug (missing s_endpgm)! +; +; GCN: s_branch [[BB:BB[0-9]+_[0-9]+]] +; GCN: [[BB]]: +; GCN-NEXT: .Lfunc_end0: +define amdgpu_ps void @ham(float %arg, float %arg1) #0 { +bb: + %tmp = fcmp ogt float %arg, 0.000000e+00 + %tmp2 = fcmp ogt float %arg1, 0.000000e+00 + %tmp3 = and i1 %tmp, %tmp2 + br i1 %tmp3, label %bb4, label %bb5 + +bb4: ; preds = %bb + unreachable + +bb5: ; preds = %bb + ret void +} + +attributes #0 = { nounwind readonly "InitialPSInputAddr"="36983" } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/branch-relax-spill.ll b/test/CodeGen/AMDGPU/branch-relax-spill.ll new file mode 100644 index 000000000000..86b8dd89e7d0 --- /dev/null +++ b/test/CodeGen/AMDGPU/branch-relax-spill.ll @@ -0,0 +1,238 @@ +; RUN: not llc -march=amdgcn -verify-machineinstrs -amdgpu-s-branch-bits=4 < %s 2>&1 | FileCheck -check-prefix=FAIL %s + +; FIXME: This should be able to compile, but requires inserting an +; extra block to restore the scavenged register. + +; FAIL: LLVM ERROR: Error while trying to spill VCC from class SReg_64: Cannot scavenge register without an emergency spill slot! + +define void @spill(i32 addrspace(1)* %arg, i32 %cnd) #0 { +entry: + %sgpr0 = tail call i32 asm sideeffect "s_mov_b32 s0, 0", "={SGPR0}"() #0 + %sgpr1 = tail call i32 asm sideeffect "s_mov_b32 s1, 0", "={SGPR1}"() #0 + %sgpr2 = tail call i32 asm sideeffect "s_mov_b32 s2, 0", "={SGPR2}"() #0 + %sgpr3 = tail call i32 asm sideeffect "s_mov_b32 s3, 0", "={SGPR3}"() #0 + %sgpr4 = tail call i32 asm sideeffect "s_mov_b32 s4, 0", "={SGPR4}"() #0 + %sgpr5 = tail call i32 asm sideeffect "s_mov_b32 s5, 0", "={SGPR5}"() #0 + %sgpr6 = tail call i32 asm sideeffect "s_mov_b32 s6, 0", "={SGPR6}"() #0 + %sgpr7 = tail call i32 asm sideeffect "s_mov_b32 s7, 0", "={SGPR7}"() #0 + %sgpr8 = tail call i32 asm sideeffect "s_mov_b32 s8, 0", "={SGPR8}"() #0 + %sgpr9 = tail call i32 asm sideeffect "s_mov_b32 s9, 0", "={SGPR9}"() #0 + %sgpr10 = tail call i32 asm sideeffect "s_mov_b32 s10, 0", "={SGPR10}"() #0 + %sgpr11 = tail call i32 asm sideeffect "s_mov_b32 s11, 0", "={SGPR11}"() #0 + %sgpr12 = tail call i32 asm sideeffect "s_mov_b32 s12, 0", "={SGPR12}"() #0 + %sgpr13 = tail call i32 asm sideeffect "s_mov_b32 s13, 0", "={SGPR13}"() #0 + %sgpr14 = tail call i32 asm sideeffect "s_mov_b32 s14, 0", "={SGPR14}"() #0 + %sgpr15 = tail call i32 asm sideeffect "s_mov_b32 s15, 0", "={SGPR15}"() #0 + %sgpr16 = tail call i32 asm sideeffect "s_mov_b32 s16, 0", "={SGPR16}"() #0 + %sgpr17 = tail call i32 asm sideeffect "s_mov_b32 s17, 0", "={SGPR17}"() #0 + %sgpr18 = tail call i32 asm sideeffect "s_mov_b32 s18, 0", "={SGPR18}"() #0 + %sgpr19 = tail call i32 asm sideeffect "s_mov_b32 s19, 0", "={SGPR19}"() #0 + %sgpr20 = tail call i32 asm sideeffect "s_mov_b32 s20, 0", "={SGPR20}"() #0 + %sgpr21 = tail call i32 asm sideeffect "s_mov_b32 s21, 0", "={SGPR21}"() #0 + %sgpr22 = tail call i32 asm sideeffect "s_mov_b32 s22, 0", "={SGPR22}"() #0 + %sgpr23 = tail call i32 asm sideeffect "s_mov_b32 s23, 0", "={SGPR23}"() #0 + %sgpr24 = tail call i32 asm sideeffect "s_mov_b32 s24, 0", "={SGPR24}"() #0 + %sgpr25 = tail call i32 asm sideeffect "s_mov_b32 s25, 0", "={SGPR25}"() #0 + %sgpr26 = tail call i32 asm sideeffect "s_mov_b32 s26, 0", "={SGPR26}"() #0 + %sgpr27 = tail call i32 asm sideeffect "s_mov_b32 s27, 0", "={SGPR27}"() #0 + %sgpr28 = tail call i32 asm sideeffect "s_mov_b32 s28, 0", "={SGPR28}"() #0 + %sgpr29 = tail call i32 asm sideeffect "s_mov_b32 s29, 0", "={SGPR29}"() #0 + %sgpr30 = tail call i32 asm sideeffect "s_mov_b32 s30, 0", "={SGPR30}"() #0 + %sgpr31 = tail call i32 asm sideeffect "s_mov_b32 s31, 0", "={SGPR31}"() #0 + %sgpr32 = tail call i32 asm sideeffect "s_mov_b32 s32, 0", "={SGPR32}"() #0 + %sgpr33 = tail call i32 asm sideeffect "s_mov_b32 s33, 0", "={SGPR33}"() #0 + %sgpr34 = tail call i32 asm sideeffect "s_mov_b32 s34, 0", "={SGPR34}"() #0 + %sgpr35 = tail call i32 asm sideeffect "s_mov_b32 s35, 0", "={SGPR35}"() #0 + %sgpr36 = tail call i32 asm sideeffect "s_mov_b32 s36, 0", "={SGPR36}"() #0 + %sgpr37 = tail call i32 asm sideeffect "s_mov_b32 s37, 0", "={SGPR37}"() #0 + %sgpr38 = tail call i32 asm sideeffect "s_mov_b32 s38, 0", "={SGPR38}"() #0 + %sgpr39 = tail call i32 asm sideeffect "s_mov_b32 s39, 0", "={SGPR39}"() #0 + %sgpr40 = tail call i32 asm sideeffect "s_mov_b32 s40, 0", "={SGPR40}"() #0 + %sgpr41 = tail call i32 asm sideeffect "s_mov_b32 s41, 0", "={SGPR41}"() #0 + %sgpr42 = tail call i32 asm sideeffect "s_mov_b32 s42, 0", "={SGPR42}"() #0 + %sgpr43 = tail call i32 asm sideeffect "s_mov_b32 s43, 0", "={SGPR43}"() #0 + %sgpr44 = tail call i32 asm sideeffect "s_mov_b32 s44, 0", "={SGPR44}"() #0 + %sgpr45 = tail call i32 asm sideeffect "s_mov_b32 s45, 0", "={SGPR45}"() #0 + %sgpr46 = tail call i32 asm sideeffect "s_mov_b32 s46, 0", "={SGPR46}"() #0 + %sgpr47 = tail call i32 asm sideeffect "s_mov_b32 s47, 0", "={SGPR47}"() #0 + %sgpr48 = tail call i32 asm sideeffect "s_mov_b32 s48, 0", "={SGPR48}"() #0 + %sgpr49 = tail call i32 asm sideeffect "s_mov_b32 s49, 0", "={SGPR49}"() #0 + %sgpr50 = tail call i32 asm sideeffect "s_mov_b32 s50, 0", "={SGPR50}"() #0 + %sgpr51 = tail call i32 asm sideeffect "s_mov_b32 s51, 0", "={SGPR51}"() #0 + %sgpr52 = tail call i32 asm sideeffect "s_mov_b32 s52, 0", "={SGPR52}"() #0 + %sgpr53 = tail call i32 asm sideeffect "s_mov_b32 s53, 0", "={SGPR53}"() #0 + %sgpr54 = tail call i32 asm sideeffect "s_mov_b32 s54, 0", "={SGPR54}"() #0 + %sgpr55 = tail call i32 asm sideeffect "s_mov_b32 s55, 0", "={SGPR55}"() #0 + %sgpr56 = tail call i32 asm sideeffect "s_mov_b32 s56, 0", "={SGPR56}"() #0 + %sgpr57 = tail call i32 asm sideeffect "s_mov_b32 s57, 0", "={SGPR57}"() #0 + %sgpr58 = tail call i32 asm sideeffect "s_mov_b32 s58, 0", "={SGPR58}"() #0 + %sgpr59 = tail call i32 asm sideeffect "s_mov_b32 s59, 0", "={SGPR59}"() #0 + %sgpr60 = tail call i32 asm sideeffect "s_mov_b32 s60, 0", "={SGPR60}"() #0 + %sgpr61 = tail call i32 asm sideeffect "s_mov_b32 s61, 0", "={SGPR61}"() #0 + %sgpr62 = tail call i32 asm sideeffect "s_mov_b32 s62, 0", "={SGPR62}"() #0 + %sgpr63 = tail call i32 asm sideeffect "s_mov_b32 s63, 0", "={SGPR63}"() #0 + %sgpr64 = tail call i32 asm sideeffect "s_mov_b32 s64, 0", "={SGPR64}"() #0 + %sgpr65 = tail call i32 asm sideeffect "s_mov_b32 s65, 0", "={SGPR65}"() #0 + %sgpr66 = tail call i32 asm sideeffect "s_mov_b32 s66, 0", "={SGPR66}"() #0 + %sgpr67 = tail call i32 asm sideeffect "s_mov_b32 s67, 0", "={SGPR67}"() #0 + %sgpr68 = tail call i32 asm sideeffect "s_mov_b32 s68, 0", "={SGPR68}"() #0 + %sgpr69 = tail call i32 asm sideeffect "s_mov_b32 s69, 0", "={SGPR69}"() #0 + %sgpr70 = tail call i32 asm sideeffect "s_mov_b32 s70, 0", "={SGPR70}"() #0 + %sgpr71 = tail call i32 asm sideeffect "s_mov_b32 s71, 0", "={SGPR71}"() #0 + %sgpr72 = tail call i32 asm sideeffect "s_mov_b32 s72, 0", "={SGPR72}"() #0 + %sgpr73 = tail call i32 asm sideeffect "s_mov_b32 s73, 0", "={SGPR73}"() #0 + %sgpr74 = tail call i32 asm sideeffect "s_mov_b32 s74, 0", "={SGPR74}"() #0 + %sgpr75 = tail call i32 asm sideeffect "s_mov_b32 s75, 0", "={SGPR75}"() #0 + %sgpr76 = tail call i32 asm sideeffect "s_mov_b32 s76, 0", "={SGPR76}"() #0 + %sgpr77 = tail call i32 asm sideeffect "s_mov_b32 s77, 0", "={SGPR77}"() #0 + %sgpr78 = tail call i32 asm sideeffect "s_mov_b32 s78, 0", "={SGPR78}"() #0 + %sgpr79 = tail call i32 asm sideeffect "s_mov_b32 s79, 0", "={SGPR79}"() #0 + %sgpr80 = tail call i32 asm sideeffect "s_mov_b32 s80, 0", "={SGPR80}"() #0 + %sgpr81 = tail call i32 asm sideeffect "s_mov_b32 s81, 0", "={SGPR81}"() #0 + %sgpr82 = tail call i32 asm sideeffect "s_mov_b32 s82, 0", "={SGPR82}"() #0 + %sgpr83 = tail call i32 asm sideeffect "s_mov_b32 s83, 0", "={SGPR83}"() #0 + %sgpr84 = tail call i32 asm sideeffect "s_mov_b32 s84, 0", "={SGPR84}"() #0 + %sgpr85 = tail call i32 asm sideeffect "s_mov_b32 s85, 0", "={SGPR85}"() #0 + %sgpr86 = tail call i32 asm sideeffect "s_mov_b32 s86, 0", "={SGPR86}"() #0 + %sgpr87 = tail call i32 asm sideeffect "s_mov_b32 s87, 0", "={SGPR87}"() #0 + %sgpr88 = tail call i32 asm sideeffect "s_mov_b32 s88, 0", "={SGPR88}"() #0 + %sgpr89 = tail call i32 asm sideeffect "s_mov_b32 s89, 0", "={SGPR89}"() #0 + %sgpr90 = tail call i32 asm sideeffect "s_mov_b32 s90, 0", "={SGPR90}"() #0 + %sgpr91 = tail call i32 asm sideeffect "s_mov_b32 s91, 0", "={SGPR91}"() #0 + %sgpr92 = tail call i32 asm sideeffect "s_mov_b32 s92, 0", "={SGPR92}"() #0 + %sgpr93 = tail call i32 asm sideeffect "s_mov_b32 s93, 0", "={SGPR93}"() #0 + %sgpr94 = tail call i32 asm sideeffect "s_mov_b32 s94, 0", "={SGPR94}"() #0 + %sgpr95 = tail call i32 asm sideeffect "s_mov_b32 s95, 0", "={SGPR95}"() #0 + %sgpr96 = tail call i32 asm sideeffect "s_mov_b32 s96, 0", "={SGPR96}"() #0 + %sgpr97 = tail call i32 asm sideeffect "s_mov_b32 s97, 0", "={SGPR97}"() #0 + %sgpr98 = tail call i32 asm sideeffect "s_mov_b32 s98, 0", "={SGPR98}"() #0 + %sgpr99 = tail call i32 asm sideeffect "s_mov_b32 s99, 0", "={SGPR99}"() #0 + %sgpr100 = tail call i32 asm sideeffect "s_mov_b32 s100, 0", "={SGPR100}"() #0 + %sgpr101 = tail call i32 asm sideeffect "s_mov_b32 s101, 0", "={SGPR101}"() #0 + %sgpr102 = tail call i32 asm sideeffect "s_mov_b32 s102, 0", "={SGPR102}"() #0 + %sgpr103 = tail call i32 asm sideeffect "s_mov_b32 s103, 0", "={SGPR103}"() #0 + %vcc_lo = tail call i32 asm sideeffect "s_mov_b32 $0, 0", "={VCC_LO}"() #0 + %vcc_hi = tail call i32 asm sideeffect "s_mov_b32 $0, 0", "={VCC_HI}"() #0 + %cmp = icmp eq i32 %cnd, 0 + br i1 %cmp, label %bb3, label %bb2 ; +8 dword branch + +bb2: ; 28 bytes + ; 24 byte asm + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64",""() #0 + br label %bb3 + +bb3: + tail call void asm sideeffect "; reg use $0", "{SGPR0}"(i32 %sgpr0) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR1}"(i32 %sgpr1) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR2}"(i32 %sgpr2) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR3}"(i32 %sgpr3) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR4}"(i32 %sgpr4) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR5}"(i32 %sgpr5) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR6}"(i32 %sgpr6) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR7}"(i32 %sgpr7) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR8}"(i32 %sgpr8) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR9}"(i32 %sgpr9) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR10}"(i32 %sgpr10) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR11}"(i32 %sgpr11) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR12}"(i32 %sgpr12) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR13}"(i32 %sgpr13) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR14}"(i32 %sgpr14) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR15}"(i32 %sgpr15) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR16}"(i32 %sgpr16) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR17}"(i32 %sgpr17) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR18}"(i32 %sgpr18) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR19}"(i32 %sgpr19) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR20}"(i32 %sgpr20) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR21}"(i32 %sgpr21) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR22}"(i32 %sgpr22) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR23}"(i32 %sgpr23) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR24}"(i32 %sgpr24) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR25}"(i32 %sgpr25) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR26}"(i32 %sgpr26) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR27}"(i32 %sgpr27) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR28}"(i32 %sgpr28) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR29}"(i32 %sgpr29) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR30}"(i32 %sgpr30) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR31}"(i32 %sgpr31) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR32}"(i32 %sgpr32) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR33}"(i32 %sgpr33) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR34}"(i32 %sgpr34) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR35}"(i32 %sgpr35) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR36}"(i32 %sgpr36) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR37}"(i32 %sgpr37) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR38}"(i32 %sgpr38) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR39}"(i32 %sgpr39) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR40}"(i32 %sgpr40) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR41}"(i32 %sgpr41) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR42}"(i32 %sgpr42) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR43}"(i32 %sgpr43) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR44}"(i32 %sgpr44) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR45}"(i32 %sgpr45) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR46}"(i32 %sgpr46) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR47}"(i32 %sgpr47) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR48}"(i32 %sgpr48) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR49}"(i32 %sgpr49) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR50}"(i32 %sgpr50) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR51}"(i32 %sgpr51) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR52}"(i32 %sgpr52) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR53}"(i32 %sgpr53) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR54}"(i32 %sgpr54) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR55}"(i32 %sgpr55) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR56}"(i32 %sgpr56) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR57}"(i32 %sgpr57) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR58}"(i32 %sgpr58) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR59}"(i32 %sgpr59) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR60}"(i32 %sgpr60) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR61}"(i32 %sgpr61) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR62}"(i32 %sgpr62) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR63}"(i32 %sgpr63) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR64}"(i32 %sgpr64) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR65}"(i32 %sgpr65) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR66}"(i32 %sgpr66) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR67}"(i32 %sgpr67) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR68}"(i32 %sgpr68) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR69}"(i32 %sgpr69) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR70}"(i32 %sgpr70) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR71}"(i32 %sgpr71) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR72}"(i32 %sgpr72) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR73}"(i32 %sgpr73) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR74}"(i32 %sgpr74) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR75}"(i32 %sgpr75) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR76}"(i32 %sgpr76) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR77}"(i32 %sgpr77) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR78}"(i32 %sgpr78) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR79}"(i32 %sgpr79) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR80}"(i32 %sgpr80) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR81}"(i32 %sgpr81) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR82}"(i32 %sgpr82) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR83}"(i32 %sgpr83) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR84}"(i32 %sgpr84) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR85}"(i32 %sgpr85) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR86}"(i32 %sgpr86) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR87}"(i32 %sgpr87) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR88}"(i32 %sgpr88) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR89}"(i32 %sgpr89) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR90}"(i32 %sgpr90) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR91}"(i32 %sgpr91) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR92}"(i32 %sgpr92) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR93}"(i32 %sgpr93) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR94}"(i32 %sgpr94) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR95}"(i32 %sgpr95) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR96}"(i32 %sgpr96) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR97}"(i32 %sgpr97) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR98}"(i32 %sgpr98) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR99}"(i32 %sgpr99) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR100}"(i32 %sgpr100) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR101}"(i32 %sgpr101) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR102}"(i32 %sgpr102) #0 + tail call void asm sideeffect "; reg use $0", "{SGPR103}"(i32 %sgpr103) #0 + tail call void asm sideeffect "; reg use $0", "{VCC_LO}"(i32 %vcc_lo) #0 + tail call void asm sideeffect "; reg use $0", "{VCC_HI}"(i32 %vcc_hi) #0 + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/branch-relaxation.ll b/test/CodeGen/AMDGPU/branch-relaxation.ll new file mode 100644 index 000000000000..39505404a868 --- /dev/null +++ b/test/CodeGen/AMDGPU/branch-relaxation.ll @@ -0,0 +1,530 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -amdgpu-s-branch-bits=4 < %s | FileCheck -check-prefix=GCN %s +; Restrict maximum branch to between +7 and -8 dwords + +; Used to emit an always 4 byte instruction. Inline asm always assumes +; each instruction is the maximum size. +declare void @llvm.amdgcn.s.sleep(i32) #0 + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + + +; GCN-LABEL: {{^}}uniform_conditional_max_short_forward_branch: +; GCN: s_load_dword [[CND:s[0-9]+]] +; GCN: s_cmp_eq_u32 [[CND]], 0 +; GCN-NEXT: s_cbranch_scc1 [[BB3:BB[0-9]+_[0-9]+]] + + +; GCN-NEXT: ; BB#1: ; %bb2 +; GCN-NEXT: ;;#ASMSTART +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: ;;#ASMEND +; GCN-NEXT: s_sleep 0 + +; GCN-NEXT: [[BB3]]: ; %bb3 +; GCN: v_mov_b32_e32 [[V_CND:v[0-9]+]], [[CND]] +; GCN: buffer_store_dword [[V_CND]] +; GCN: s_endpgm +define void @uniform_conditional_max_short_forward_branch(i32 addrspace(1)* %arg, i32 %cnd) #0 { +bb: + %cmp = icmp eq i32 %cnd, 0 + br i1 %cmp, label %bb3, label %bb2 ; +8 dword branch + +bb2: +; 24 bytes + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + call void @llvm.amdgcn.s.sleep(i32 0) + br label %bb3 + +bb3: + store volatile i32 %cnd, i32 addrspace(1)* %arg + ret void +} + +; GCN-LABEL: {{^}}uniform_conditional_min_long_forward_branch: +; GCN: s_load_dword [[CND:s[0-9]+]] +; GCN: s_cmp_eq_u32 [[CND]], 0 +; GCN-NEXT: s_cbranch_scc0 [[LONGBB:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONG_JUMP:BB[0-9]+_[0-9]+]]: ; %bb0 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[ENDBB:BB[0-9]+_[0-9]+]]-([[LONG_JUMP]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0 +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[LONGBB]]: +; GCN-NEXT: ;;#ASMSTART +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN-NEXT: ;;#ASMEND + +; GCN-NEXT: [[ENDBB]]: +; GCN: v_mov_b32_e32 [[V_CND:v[0-9]+]], [[CND]] +; GCN: buffer_store_dword [[V_CND]] +; GCN: s_endpgm +define void @uniform_conditional_min_long_forward_branch(i32 addrspace(1)* %arg, i32 %cnd) #0 { +bb0: + %cmp = icmp eq i32 %cnd, 0 + br i1 %cmp, label %bb3, label %bb2 ; +9 dword branch + +bb2: +; 32 bytes + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %bb3 + +bb3: + store volatile i32 %cnd, i32 addrspace(1)* %arg + ret void +} + +; GCN-LABEL: {{^}}uniform_conditional_min_long_forward_vcnd_branch: +; GCN: s_load_dword [[CND:s[0-9]+]] +; GCN-DAG: v_mov_b32_e32 [[V_CND:v[0-9]+]], [[CND]] +; GCN-DAG: v_cmp_eq_f32_e64 vcc, [[CND]], 0 +; GCN: s_cbranch_vccz [[LONGBB:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONG_JUMP:BB[0-9]+_[0-9]+]]: ; %bb0 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[ENDBB:BB[0-9]+_[0-9]+]]-([[LONG_JUMP]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0 +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[LONGBB]]: +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 + +; GCN: [[ENDBB]]: +; GCN: buffer_store_dword [[V_CND]] +; GCN: s_endpgm +define void @uniform_conditional_min_long_forward_vcnd_branch(float addrspace(1)* %arg, float %cnd) #0 { +bb0: + %cmp = fcmp oeq float %cnd, 0.0 + br i1 %cmp, label %bb3, label %bb2 ; + 8 dword branch + +bb2: + call void asm sideeffect " ; 32 bytes + v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %bb3 + +bb3: + store volatile float %cnd, float addrspace(1)* %arg + ret void +} + +; GCN-LABEL: {{^}}min_long_forward_vbranch: + +; GCN: buffer_load_dword +; GCN: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}} +; GCN: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], vcc +; GCN: s_xor_b64 [[SAVE]], exec, [[SAVE]] + +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 + +; GCN: s_or_b64 exec, exec, [[SAVE]] +; GCN: buffer_store_dword +; GCN: s_endpgm +define void @min_long_forward_vbranch(i32 addrspace(1)* %arg) #0 { +bb: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = zext i32 %tid to i64 + %gep = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tid.ext + %load = load volatile i32, i32 addrspace(1)* %gep + %cmp = icmp eq i32 %load, 0 + br i1 %cmp, label %bb3, label %bb2 ; + 8 dword branch + +bb2: + call void asm sideeffect " ; 32 bytes + v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %bb3 + +bb3: + store volatile i32 %load, i32 addrspace(1)* %gep + ret void +} + +; GCN-LABEL: {{^}}long_backward_sbranch: +; GCN: s_mov_b32 [[LOOPIDX:s[0-9]+]], 0{{$}} + +; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: ; %bb2 +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: s_add_i32 [[INC:s[0-9]+]], [[LOOPIDX]], 1 +; GCN-NEXT: s_cmp_lt_i32 [[INC]], 10 + +; GCN-NEXT: ;;#ASMSTART +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: ;;#ASMEND + +; GCN-NEXT: s_cbranch_scc0 [[ENDBB:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONG_JUMP:BB[0-9]+_[0-9]+]]: ; %bb2 +; GCN-NEXT: ; in Loop: Header=[[LOOPBB]] Depth=1 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONG_JUMP]]+4)-[[LOOPBB]] +; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0 +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[ENDBB]]: +; GCN-NEXT: s_endpgm +define void @long_backward_sbranch(i32 addrspace(1)* %arg) #0 { +bb: + br label %bb2 + +bb2: + %loop.idx = phi i32 [ 0, %bb ], [ %inc, %bb2 ] + ; 24 bytes + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + %inc = add nsw i32 %loop.idx, 1 ; add cost 4 + %cmp = icmp slt i32 %inc, 10 ; condition cost = 8 + br i1 %cmp, label %bb2, label %bb3 ; - + +bb3: + ret void +} + +; Requires expansion of unconditional branch from %bb2 to %bb4 (and +; expansion of conditional branch from %bb to %bb3. + +; GCN-LABEL: {{^}}uniform_unconditional_min_long_forward_branch: +; GCN: s_cmp_eq_u32 +; GCN-NEXT: s_cbranch_scc0 [[BB2:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONG_JUMP0:BB[0-9]+_[0-9]+]]: ; %bb0 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[BB3:BB[0-9]_[0-9]+]]-([[LONG_JUMP0]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[BB2]]: ; %bb2 +; GCN: v_mov_b32_e32 [[BB2_K:v[0-9]+]], 17 +; GCN: buffer_store_dword [[BB2_K]] +; GCN: s_waitcnt vmcnt(0) + +; GCN-NEXT: [[LONG_JUMP1:BB[0-9]+_[0-9]+]]: ; %bb2 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[BB4:BB[0-9]_[0-9]+]]-([[LONG_JUMP1]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc + +; GCN: [[BB3]]: ; %bb3 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: ;;#ASMEND + +; GCN-NEXT: [[BB4]]: ; %bb4 +; GCN: v_mov_b32_e32 [[BB4_K:v[0-9]+]], 63 +; GCN: buffer_store_dword [[BB4_K]] +; GCN-NEXT: s_endpgm +; GCN-NEXT: .Lfunc_end{{[0-9]+}}: +define void @uniform_unconditional_min_long_forward_branch(i32 addrspace(1)* %arg, i32 %arg1) { +bb0: + %tmp = icmp ne i32 %arg1, 0 + br i1 %tmp, label %bb2, label %bb3 + +bb2: + store volatile i32 17, i32 addrspace(1)* undef + br label %bb4 + +bb3: + ; 32 byte asm + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %bb4 + +bb4: + store volatile i32 63, i32 addrspace(1)* %arg + ret void +} + +; GCN-LABEL: {{^}}uniform_unconditional_min_long_backward_branch: +; GCN-NEXT: ; BB#0: ; %entry + +; GCN-NEXT: [[LOOP:BB[0-9]_[0-9]+]]: ; %loop +; GCN-NEXT: ; =>This Inner Loop Header: Depth=1 +; GCN-NEXT: ;;#ASMSTART +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: ;;#ASMEND + +; GCN-NEXT: [[LONGBB:BB[0-9]+_[0-9]+]]: ; %loop +; GCN-NEXT: ; in Loop: Header=[[LOOP]] Depth=1 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONGBB]]+4)-[[LOOP]] +; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc +; GCN-NEXT .Lfunc_end{{[0-9]+}}: +define void @uniform_unconditional_min_long_backward_branch(i32 addrspace(1)* %arg, i32 %arg1) { +entry: + br label %loop + +loop: + ; 32 byte asm + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %loop +} + +; Expansion of branch from %bb1 to %bb3 introduces need to expand +; branch from %bb0 to %bb2 + +; GCN-LABEL: {{^}}expand_requires_expand: +; GCN-NEXT: ; BB#0: ; %bb0 +; GCN: s_load_dword +; GCN: s_cmp_lt_i32 s{{[0-9]+}}, 0{{$}} +; GCN-NEXT: s_cbranch_scc0 [[BB1:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONGBB0:BB[0-9]+_[0-9]+]]: ; %bb0 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[BB2:BB[0-9]_[0-9]+]]-([[LONGBB0]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[BB1]]: ; %bb1 +; GCN-NEXT: s_load_dword +; GCN-NEXT: s_waitcnt lgkmcnt(0) +; GCN-NEXT: s_cmp_eq_u32 s{{[0-9]+}}, 3{{$}} +; GCN-NEXT: s_cbranch_scc0 [[BB2:BB[0-9]_[0-9]+]] + +; GCN-NEXT: [[LONGBB1:BB[0-9]+_[0-9]+]]: ; %bb1 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[BB3:BB[0-9]+_[0-9]+]]-([[LONGBB1]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[BB2]]: ; %bb2 +; GCN-NEXT: ;;#ASMSTART +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: v_nop_e64 +; GCN-NEXT: ;;#ASMEND + +; GCN-NEXT: [[BB3]]: ; %bb3 +; GCN-NEXT: s_endpgm +define void @expand_requires_expand(i32 %cond0) #0 { +bb0: + %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %cmp0 = icmp slt i32 %cond0, 0 + br i1 %cmp0, label %bb2, label %bb1 + +bb1: + %val = load volatile i32, i32 addrspace(2)* undef + %cmp1 = icmp eq i32 %val, 3 + br i1 %cmp1, label %bb3, label %bb2 + +bb2: + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %bb3 + +bb3: + ret void +} + +; Requires expanding of required skip branch. + +; GCN-LABEL: {{^}}uniform_inside_divergent: +; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} +; GCN-NEXT: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc +; GCN-NEXT: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] +; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9]+_[0-9]+]] +; GCN-NEXT: s_cbranch_execnz [[IF:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[LONGBB:BB[0-9]+_[0-9]+]]: ; %entry +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[BB2:BB[0-9]_[0-9]+]]-([[LONGBB]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0{{$}} +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[IF]]: ; %if +; GCN: buffer_store_dword +; GCN: s_cmp_lg_u32 +; GCN: s_cbranch_scc1 [[ENDIF]] + +; GCN-NEXT: ; BB#2: ; %if_uniform +; GCN: buffer_store_dword +; GCN: s_waitcnt vmcnt(0) + +; GCN-NEXT: [[ENDIF]]: ; %endif +; GCN-NEXT: s_or_b64 exec, exec, [[MASK]] +; GCN-NEXT: s_endpgm +define void @uniform_inside_divergent(i32 addrspace(1)* %out, i32 %cond) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %d_cmp = icmp ult i32 %tid, 16 + br i1 %d_cmp, label %if, label %endif + +if: + store i32 0, i32 addrspace(1)* %out + %u_cmp = icmp eq i32 %cond, 0 + br i1 %u_cmp, label %if_uniform, label %endif + +if_uniform: + store i32 1, i32 addrspace(1)* %out + br label %endif + +endif: + ret void +} + +; si_mask_branch +; s_cbranch_execz +; s_branch + +; GCN-LABEL: {{^}}analyze_mask_branch: +; GCN: v_cmp_lt_f32_e32 vcc +; GCN-NEXT: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc +; GCN-NEXT: s_xor_b64 [[MASK]], exec, [[MASK]] +; GCN-NEXT: ; mask branch [[RET:BB[0-9]+_[0-9]+]] +; GCN-NEXT: s_cbranch_execz [[BRANCH_SKIP:BB[0-9]+_[0-9]+]] +; GCN-NEXT: s_branch [[LOOP_BODY:BB[0-9]+_[0-9]+]] + +; GCN-NEXT: [[BRANCH_SKIP]]: ; %entry +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_add_u32 vcc_lo, vcc_lo, [[RET]]-([[BRANCH_SKIP]]+4) +; GCN-NEXT: s_addc_u32 vcc_hi, vcc_hi, 0 +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[LOOP_BODY]]: ; %loop_body +; GCN: s_mov_b64 vcc, -1{{$}} +; GCN: ;;#ASMSTART +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: v_nop_e64 +; GCN: ;;#ASMEND +; GCN-NEXT: s_cbranch_vccz [[RET]] + +; GCN-NEXT: [[LONGBB:BB[0-9]+_[0-9]+]]: ; %loop_body +; GCN-NEXT: ; in Loop: Header=[[LOOP_BODY]] Depth=1 +; GCN-NEXT: s_getpc_b64 vcc +; GCN-NEXT: s_sub_u32 vcc_lo, vcc_lo, ([[LONGBB]]+4)-[[LOOP_BODY]] +; GCN-NEXT: s_subb_u32 vcc_hi, vcc_hi, 0 +; GCN-NEXT: s_setpc_b64 vcc + +; GCN-NEXT: [[RET]]: ; %Flow +; GCN-NEXT: s_or_b64 exec, exec, [[MASK]] +; GCN: buffer_store_dword +; GCN-NEXT: s_endpgm +define void @analyze_mask_branch() #0 { +entry: + %reg = call float asm sideeffect "v_mov_b32_e64 $0, 0", "=v"() + %cmp0 = fcmp ogt float %reg, 0.000000e+00 + br i1 %cmp0, label %loop, label %ret + +loop: + %phi = phi float [ 0.000000e+00, %loop_body ], [ 1.000000e+00, %entry ] + call void asm sideeffect + "v_nop_e64 + v_nop_e64", ""() #0 + %cmp1 = fcmp olt float %phi, 8.0 + br i1 %cmp1, label %loop_body, label %ret + +loop_body: + call void asm sideeffect + "v_nop_e64 + v_nop_e64 + v_nop_e64 + v_nop_e64", ""() #0 + br label %loop + +ret: + store volatile i32 7, i32 addrspace(1)* undef + ret void +} + +; GCN-LABEL: {{^}}long_branch_hang: +; GCN: s_cmp_lt_i32 s{{[0-9]+}}, 6 +; GCN-NEXT: s_cbranch_scc0 [[LONG_BR_0:BB[0-9]+_[0-9]+]] +; GCN-NEXT: BB{{[0-9]+_[0-9]+}}: + +; GCN: s_add_u32 vcc_lo, vcc_lo, [[LONG_BR_DEST0:BB[0-9]+_[0-9]+]]-( +; GCN: s_setpc_b64 + +; GCN-NEXT: [[LONG_BR_0]]: +; GCN-DAG: v_cmp_lt_i32 +; GCN-DAG: v_cmp_gt_i32 +; GCN: s_cbranch_vccnz + +; GCN: s_setpc_b64 +; GCN: s_setpc_b64 + +; GCN: [[LONG_BR_DEST0]] +; GCN: v_cmp_ne_u32_e32 +; GCN-NEXT: s_cbranch_vccz +; GCN: s_setpc_b64 + +; GCN: s_endpgm +define amdgpu_kernel void @long_branch_hang(i32 addrspace(1)* nocapture %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i64 %arg5) #0 { +bb: + %tmp = icmp slt i32 %arg2, 9 + %tmp6 = icmp eq i32 %arg1, 0 + %tmp7 = icmp sgt i32 %arg4, 0 + %tmp8 = icmp sgt i32 %arg4, 5 + br i1 %tmp8, label %bb9, label %bb13 + +bb9: ; preds = %bb + %tmp10 = and i1 %tmp7, %tmp + %tmp11 = icmp slt i32 %arg3, %arg4 + %tmp12 = or i1 %tmp11, %tmp7 + br i1 %tmp12, label %bb19, label %bb14 + +bb13: ; preds = %bb + br i1 %tmp6, label %bb19, label %bb14 + +bb14: ; preds = %bb13, %bb9 + %tmp15 = icmp slt i32 %arg3, %arg4 + %tmp16 = or i1 %tmp15, %tmp + %tmp17 = and i1 %tmp6, %tmp16 + %tmp18 = zext i1 %tmp17 to i32 + br label %bb19 + +bb19: ; preds = %bb14, %bb13, %bb9 + %tmp20 = phi i32 [ undef, %bb9 ], [ undef, %bb13 ], [ %tmp18, %bb14 ] + %tmp21 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %arg5 + store i32 %tmp20, i32 addrspace(1)* %tmp21, align 4 + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/branch-uniformity.ll b/test/CodeGen/AMDGPU/branch-uniformity.ll index d1a1f93f0210..e6f684178035 100644 --- a/test/CodeGen/AMDGPU/branch-uniformity.ll +++ b/test/CodeGen/AMDGPU/branch-uniformity.ll @@ -1,4 +1,4 @@ -; RUN: llc -mtriple=amdgcn-- < %s | FileCheck %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s ; The branch instruction in LOOP49 has a uniform condition, but PHI instructions ; introduced by the structurizecfg pass previously caused a false divergence @@ -8,7 +8,7 @@ ; ; CHECK-LABEL: {{^}}main: ; CHECK: ; %LOOP49 -; CHECK: v_cmp_ne_i32_e32 vcc, +; CHECK: v_cmp_ne_u32_e32 vcc, ; CHECK: s_cbranch_vccnz ; CHECK: ; %ENDIF53 define amdgpu_vs float @main(i32 %in) { diff --git a/test/CodeGen/AMDGPU/bswap.ll b/test/CodeGen/AMDGPU/bswap.ll index 4cf8e4bfed50..23b93ce2f075 100644 --- a/test/CodeGen/AMDGPU/bswap.ll +++ b/test/CodeGen/AMDGPU/bswap.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s declare i32 @llvm.bswap.i32(i32) nounwind readnone @@ -93,6 +93,8 @@ define void @test_bswap_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> addrspace( ret void } +; FUNC-LABEL: {{^}}test_bswap_i64: +; SI-NOT: v_or_b32_e64 v{{[0-9]+}}, 0, 0 define void @test_bswap_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) nounwind { %val = load i64, i64 addrspace(1)* %in, align 8 %bswap = call i64 @llvm.bswap.i64(i64 %val) nounwind readnone diff --git a/test/CodeGen/AMDGPU/call.ll b/test/CodeGen/AMDGPU/call.ll index a3e986d367e1..6d101e1537cc 100644 --- a/test/CodeGen/AMDGPU/call.ll +++ b/test/CodeGen/AMDGPU/call.ll @@ -5,6 +5,7 @@ ; CHECK: in function test_call_external{{.*}}: unsupported call to function external_function ; CHECK: in function test_call{{.*}}: unsupported call to function defined_function ; CHECK: in function test_tail_call{{.*}}: unsupported call to function defined_function +; CHECK: in function test_tail_call_bitcast_extern_variadic{{.*}}: unsupported call to function extern_variadic declare i32 @external_function(i32) nounwind @@ -43,3 +44,16 @@ define void @test_tail_call(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { store i32 %result, i32 addrspace(1)* %out ret void } + +define i32 @test_tail_call_ret() { + %call = call i32 @external_function(i32 10) + ret i32 %call +} + +declare i32 @extern_variadic(...) + +define i32 @test_tail_call_bitcast_extern_variadic(<4 x float> %arg0, <4 x float> %arg1, i32 %arg2) { + %add = fadd <4 x float> %arg0, %arg1 + %call = tail call i32 bitcast (i32 (...)* @extern_variadic to i32 (<4 x float>)*)(<4 x float> %add) #7 + ret i32 %call +} diff --git a/test/CodeGen/AMDGPU/captured-frame-index.ll b/test/CodeGen/AMDGPU/captured-frame-index.ll index 161c46b486eb..49af159581f7 100644 --- a/test/CodeGen/AMDGPU/captured-frame-index.ll +++ b/test/CodeGen/AMDGPU/captured-frame-index.ll @@ -1,9 +1,20 @@ -; RUN: llc -march=amdgcn -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -mtriple=amdgcn-- -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}store_fi_lifetime: +; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}} +; GCN: buffer_store_dword [[FI]] +define void @store_fi_lifetime(i32 addrspace(1)* %out, i32 %in) #0 { +entry: + %b = alloca i8 + call void @llvm.lifetime.start(i64 1, i8* %b) + store volatile i8* %b, i8* addrspace(1)* undef + call void @llvm.lifetime.end(i64 1, i8* %b) + ret void +} ; GCN-LABEL: {{^}}stored_fi_to_lds: ; GCN: s_load_dword [[LDSPTR:s[0-9]+]] -; GCN: v_mov_b32_e32 [[ZERO1:v[0-9]+]], 0{{$}} -; GCN: buffer_store_dword v{{[0-9]+}}, [[ZERO1]] +; GCN: buffer_store_dword v{{[0-9]+}}, off, ; GCN: v_mov_b32_e32 [[ZERO0:v[0-9]+]], 0{{$}} ; GCN: v_mov_b32_e32 [[VLDSPTR:v[0-9]+]], [[LDSPTR]] ; GCN: ds_write_b32 [[VLDSPTR]], [[ZERO0]] @@ -17,8 +28,8 @@ define void @stored_fi_to_lds(float* addrspace(3)* %ptr) #0 { ; Offset is applied ; GCN-LABEL: {{^}}stored_fi_to_lds_2_small_objects: ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} -; GCN-DAG: buffer_store_dword v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} -; GCN-DAG: buffer_store_dword v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:4{{$}} +; GCN-DAG: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} +; GCN-DAG: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}} ; GCN-DAG: s_load_dword [[LDSPTR:s[0-9]+]] @@ -40,9 +51,9 @@ define void @stored_fi_to_lds_2_small_objects(float* addrspace(3)* %ptr) #0 { ; Same frame index is used multiple times in the store ; GCN-LABEL: {{^}}stored_fi_to_self: ; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x4d2{{$}} +; GCN: buffer_store_dword [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} -; GCN: buffer_store_dword [[K]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} -; GCN: buffer_store_dword [[ZERO]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN: buffer_store_dword [[ZERO]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} define void @stored_fi_to_self() #0 { %tmp = alloca i32* @@ -54,15 +65,14 @@ define void @stored_fi_to_self() #0 { } ; GCN-LABEL: {{^}}stored_fi_to_self_offset: -; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} ; GCN-DAG: v_mov_b32_e32 [[K0:v[0-9]+]], 32{{$}} -; GCN: buffer_store_dword [[K0]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN: buffer_store_dword [[K0]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN-DAG: v_mov_b32_e32 [[K1:v[0-9]+]], 0x4d2{{$}} -; GCN: buffer_store_dword [[K1]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:2048{{$}} +; GCN: buffer_store_dword [[K1]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2048{{$}} ; GCN: v_mov_b32_e32 [[OFFSETK:v[0-9]+]], 0x800{{$}} -; GCN: buffer_store_dword [[OFFSETK]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:2048{{$}} +; GCN: buffer_store_dword [[OFFSETK]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2048{{$}} define void @stored_fi_to_self_offset() #0 { %tmp0 = alloca [512 x i32] %tmp1 = alloca i32* @@ -79,16 +89,15 @@ define void @stored_fi_to_self_offset() #0 { } ; GCN-LABEL: {{^}}stored_fi_to_fi: -; GCN: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} -; GCN: buffer_store_dword v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} -; GCN: buffer_store_dword v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:4{{$}} -; GCN: buffer_store_dword v{{[0-9]+}}, [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:8{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:8{{$}} ; GCN: v_mov_b32_e32 [[FI1:v[0-9]+]], 4{{$}} -; GCN: buffer_store_dword [[FI1]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:8{{$}} +; GCN: buffer_store_dword [[FI1]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:8{{$}} ; GCN: v_mov_b32_e32 [[FI2:v[0-9]+]], 8{{$}} -; GCN: buffer_store_dword [[FI2]], [[ZERO]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:4{{$}} +; GCN: buffer_store_dword [[FI2]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}} define void @stored_fi_to_fi() #0 { %tmp0 = alloca i32* %tmp1 = alloca i32* @@ -106,7 +115,7 @@ define void @stored_fi_to_fi() #0 { } ; GCN-LABEL: {{^}}stored_fi_to_global: -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}} ; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}} ; GCN: buffer_store_dword [[FI]] define void @stored_fi_to_global(float* addrspace(1)* %ptr) #0 { @@ -118,9 +127,9 @@ define void @stored_fi_to_global(float* addrspace(1)* %ptr) #0 { ; Offset is applied ; GCN-LABEL: {{^}}stored_fi_to_global_2_small_objects: -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4{{$}} +; GCN: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:8{{$}} ; GCN: v_mov_b32_e32 [[FI1:v[0-9]+]], 4{{$}} ; GCN: buffer_store_dword [[FI1]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} @@ -140,17 +149,20 @@ define void @stored_fi_to_global_2_small_objects(float* addrspace(1)* %ptr) #0 { } ; GCN-LABEL: {{^}}stored_fi_to_global_huge_frame_offset: -; GCN: s_add_i32 [[BASE_1_OFF_0:s[0-9]+]], 0, 0x3ffc ; GCN: v_mov_b32_e32 [[BASE_0:v[0-9]+]], 0{{$}} -; GCN: buffer_store_dword [[BASE_0]], v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen +; GCN: buffer_store_dword [[BASE_0]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}} + +; FIXME: Re-initialize +; GCN: v_mov_b32_e32 [[BASE_0_1:v[0-9]+]], 0{{$}} + +; GCN-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}} +; GCN-DAG: v_add_i32_e32 [[BASE_1_OFF_1:v[0-9]+]], vcc, 0x3ffc, [[BASE_0_1]] + -; GCN: v_mov_b32_e32 [[V_BASE_1_OFF_0:v[0-9]+]], [[BASE_1_OFF_0]] -; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3e7{{$}} -; GCN: s_add_i32 [[BASE_1_OFF_1:s[0-9]+]], 0, 56 -; GCN: buffer_store_dword [[K]], [[V_BASE_1_OFF_0]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN: v_add_i32_e32 [[BASE_1_OFF_2:v[0-9]+]], vcc, 56, [[BASE_0_1]] +; GCN: buffer_store_dword [[K]], [[BASE_1_OFF_1]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} -; GCN: v_mov_b32_e32 [[V_BASE_1_OFF_1:v[0-9]+]], [[BASE_1_OFF_1]] -; GCN: buffer_store_dword [[V_BASE_1_OFF_1]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; GCN: buffer_store_dword [[BASE_1_OFF_2]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} define void @stored_fi_to_global_huge_frame_offset(i32* addrspace(1)* %ptr) #0 { %tmp0 = alloca [4096 x i32] %tmp1 = alloca [4096 x i32] @@ -163,4 +175,29 @@ define void @stored_fi_to_global_huge_frame_offset(i32* addrspace(1)* %ptr) #0 { ret void } +@g1 = external addrspace(1) global i32* + +; This was leaving a dead node around resulting in failing to select +; on the leftover AssertZext's ValueType operand. + +; GCN-LABEL: {{^}}cannot_select_assertzext_valuetype: +; GCN: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} +; GCN: s_add_u32 s{{[0-9]+}}, s[[PC_LO]], g1@gotpcrel32@lo+4 +; GCN: s_addc_u32 s{{[0-9]+}}, s[[PC_HI]], g1@gotpcrel32@hi+4 +; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}} +; GCN: buffer_store_dword [[FI]] +define void @cannot_select_assertzext_valuetype(i32 addrspace(1)* %out, i32 %idx) #0 { +entry: + %b = alloca i32, align 4 + %tmp1 = load volatile i32*, i32* addrspace(1)* @g1, align 4 + %arrayidx = getelementptr inbounds i32, i32* %tmp1, i32 %idx + %tmp2 = load i32, i32* %arrayidx, align 4 + store volatile i32* %b, i32* addrspace(1)* undef + ret void +} + +declare void @llvm.lifetime.start(i64, i8* nocapture) #1 +declare void @llvm.lifetime.end(i64, i8* nocapture) #1 + attributes #0 = { nounwind } +attributes #1 = { argmemonly nounwind } diff --git a/test/CodeGen/AMDGPU/cf-loop-on-constant.ll b/test/CodeGen/AMDGPU/cf-loop-on-constant.ll index 759c48b3a9cf..0d919bbf85e3 100644 --- a/test/CodeGen/AMDGPU/cf-loop-on-constant.ll +++ b/test/CodeGen/AMDGPU/cf-loop-on-constant.ll @@ -95,10 +95,12 @@ for.body: ; GCN-LABEL: {{^}}loop_arg_0: ; GCN: v_and_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}} -; GCN: v_cmp_eq_i32_e32 vcc, 1, +; GCN: v_cmp_eq_u32_e32 vcc, 1, -; GCN: s_and_b64 s{{\[[0-9]+:[0-9]+\]}}, exec, vcc ; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]] +; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80 +; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 4 + ; GCN: s_cbranch_vccnz [[LOOPBB]] ; GCN-NEXT: ; BB#2 ; GCN-NEXT: s_endpgm diff --git a/test/CodeGen/AMDGPU/cgp-addressing-modes.ll b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll index 916d667ec492..f3425ca9fdd5 100644 --- a/test/CodeGen/AMDGPU/cgp-addressing-modes.ll +++ b/test/CodeGen/AMDGPU/cgp-addressing-modes.ll @@ -126,8 +126,8 @@ done: ; GCN-LABEL: {{^}}test_sink_scratch_small_offset_i32: ; GCN: s_and_saveexec_b64 -; GCN: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}} -; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:4092{{$}} +; GCN: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4092{{$}} +; GCN: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4092{{$}} ; GCN: {{^}}BB4_2: define void @test_sink_scratch_small_offset_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in, i32 %arg) { entry: @@ -220,11 +220,6 @@ done: ret void } -attributes #0 = { nounwind readnone } -attributes #1 = { nounwind } - - - ; OPT-LABEL: @test_sink_constant_small_offset_i32 ; OPT-NOT: getelementptr i32, i32 addrspace(2)* ; OPT: br i1 @@ -475,6 +470,37 @@ bb34: unreachable } +; Address offset is not a multiple of 4. This is a valid mubuf offset, +; but not smrd. + +; OPT-LABEL: @test_sink_constant_small_max_mubuf_offset_load_i32_align_1( +; OPT: br i1 %tmp0, +; OPT: if: +; OPT: %sunkaddr = ptrtoint i8 addrspace(2)* %in to i64 +; OPT: %sunkaddr1 = add i64 %sunkaddr, 4095 +define void @test_sink_constant_small_max_mubuf_offset_load_i32_align_1(i32 addrspace(1)* %out, i8 addrspace(2)* %in) { +entry: + %out.gep = getelementptr i32, i32 addrspace(1)* %out, i32 1024 + %in.gep = getelementptr i8, i8 addrspace(2)* %in, i64 4095 + %tid = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %tmp0 = icmp eq i32 %tid, 0 + br i1 %tmp0, label %endif, label %if + +if: + %bitcast = bitcast i8 addrspace(2)* %in.gep to i32 addrspace(2)* + %tmp1 = load i32, i32 addrspace(2)* %bitcast, align 1 + br label %endif + +endif: + %x = phi i32 [ %tmp1, %if ], [ 0, %entry ] + store i32 %x, i32 addrspace(1)* %out.gep + br label %done + +done: + ret void +} + declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0 attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/cgp-bitfield-extract.ll b/test/CodeGen/AMDGPU/cgp-bitfield-extract.ll index 33daf0292ae1..7aa6a53e9025 100644 --- a/test/CodeGen/AMDGPU/cgp-bitfield-extract.ll +++ b/test/CodeGen/AMDGPU/cgp-bitfield-extract.ll @@ -27,7 +27,7 @@ ; GCN-LABEL: {{^}}sink_ubfe_i32: ; GCN-NOT: lshr -; GCN: s_cbranch_vccnz +; GCN: s_cbranch_scc1 ; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80008 ; GCN: BB0_2: @@ -116,14 +116,19 @@ ret: ; OPT: store ; OPT: ret +; For GFX8: since i16 is legal type, we cannot sink lshr into BBs. ; GCN-LABEL: {{^}}sink_ubfe_i16: ; GCN-NOT: lshr -; GCN: s_cbranch_vccnz +; VI: s_bfe_u32 s0, s0, 0xc0004 +; GCN: s_cbranch_scc1 + +; SI: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80004 +; VI: s_and_b32 s0, s0, 0xff -; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80004 ; GCN: BB2_2: -; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x70004 +; SI: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x70004 +; VI: s_and_b32 s0, s0, 0x7f ; GCN: BB2_3: ; GCN: buffer_store_short @@ -170,7 +175,7 @@ ret: ; OPT: ret ; GCN-LABEL: {{^}}sink_ubfe_i64_span_midpoint: -; GCN: s_cbranch_vccnz BB3_2 +; GCN: s_cbranch_scc1 BB3_2 ; GCN: s_lshr_b64 s{{\[}}[[LO:[0-9]+]]:{{[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}}, 30 ; GCN: s_and_b32 s{{[0-9]+}}, s[[LO]], 0xff @@ -221,7 +226,7 @@ ret: ; GCN-LABEL: {{^}}sink_ubfe_i64_low32: -; GCN: s_cbranch_vccnz BB4_2 +; GCN: s_cbranch_scc1 BB4_2 ; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x8000f @@ -269,7 +274,7 @@ ret: ; OPT: ret ; GCN-LABEL: {{^}}sink_ubfe_i64_high32: -; GCN: s_cbranch_vccnz BB5_2 +; GCN: s_cbranch_scc1 BB5_2 ; GCN: s_bfe_u32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80003 ; GCN: BB5_2: diff --git a/test/CodeGen/AMDGPU/cndmask-no-def-vcc.ll b/test/CodeGen/AMDGPU/cndmask-no-def-vcc.ll index 2c4c07c193af..9b4b61cf728a 100644 --- a/test/CodeGen/AMDGPU/cndmask-no-def-vcc.ll +++ b/test/CodeGen/AMDGPU/cndmask-no-def-vcc.ll @@ -5,9 +5,9 @@ declare i1 @llvm.amdgcn.class.f32(float, i32) ; Produces error after adding an implicit def to v_cndmask_b32 ; GCN-LABEL: {{^}}vcc_shrink_vcc_def: -; GCN: v_cmp_eq_i32_e64 vcc, 0, s{{[0-9]+}} +; GCN: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}} ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc -; GCN: v_cndmask_b32_e64 v1, 0, 1, s{{\[[0-9]+:[0-9]+\]}} +; GCN: v_cndmask_b32_e64 v0, 0, 1, s{{\[[0-9]+:[0-9]+\]}} define void @vcc_shrink_vcc_def(float %arg, i32 %arg1, float %arg2, i32 %arg3) { bb0: %tmp = icmp sgt i32 %arg1, 4 @@ -34,7 +34,7 @@ bb2: ; GCN-LABEL: {{^}}preserve_condition_undef_flag: ; GCN-NOT: vcc ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc -; GCN: v_cndmask_b32_e64 v1, 0, 1, s{{\[[0-9]+:[0-9]+\]}} +; GCN: v_cndmask_b32_e64 v0, 0, 1, s{{\[[0-9]+:[0-9]+\]}} define void @preserve_condition_undef_flag(float %arg, i32 %arg1, float %arg2) { bb0: %tmp = icmp sgt i32 %arg1, 4 diff --git a/test/CodeGen/AMDGPU/coalescer-subrange-crash.ll b/test/CodeGen/AMDGPU/coalescer-subrange-crash.ll new file mode 100644 index 000000000000..7ff133b86e72 --- /dev/null +++ b/test/CodeGen/AMDGPU/coalescer-subrange-crash.ll @@ -0,0 +1,62 @@ +; RUN: llc -march=amdgcn < %s | FileCheck %s +; REQUIRES: asserts +; +; This testcase used to cause the following crash: +; +; *** Couldn't join subrange! +; +; UNREACHABLE executed at lib/CodeGen/RegisterCoalescer.cpp:2666! +; +; The insertelement instructions became subregister definitions: one virtual +; register was defined and re-defined by one group of the consecutive insert- +; elements, and another was defined by the second group. +; Since a copy between the two full registers was present in the program, +; the coalescer tried to merge them. The join algorithm for the main range +; decided that it was correct to do so, while the subrange join unexpectedly +; failed. This was caused by the live interval subranges not being computed +; correctly: subregister defs are not uses for the purpose of subranges. +; +; Test for a valid output: +; CHECK: image_sample_c_d_o + +target triple = "amdgcn--" + +define amdgpu_ps <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> @main([17 x <16 x i8>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg, [16 x <16 x i8>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg1, [32 x <8 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg2, [16 x <8 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg3, [16 x <4 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg4, float inreg %arg5, i32 inreg %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <3 x i32> %arg10, <2 x i32> %arg11, <2 x i32> %arg12, <2 x i32> %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, i32 %arg19, i32 %arg20, float %arg21, i32 %arg22) #0 { +main_body: + %tmp = call float @llvm.SI.fs.interp(i32 3, i32 0, i32 %arg6, <2 x i32> %arg8) + %tmp23 = fadd float %tmp, 0xBFA99999A0000000 + %tmp24 = fadd float %tmp, 0x3FA99999A0000000 + %tmp25 = bitcast float %tmp23 to i32 + %tmp26 = insertelement <16 x i32> <i32 212739, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, i32 %tmp25, i32 1 + %tmp27 = insertelement <16 x i32> %tmp26, i32 undef, i32 2 + %tmp28 = insertelement <16 x i32> %tmp27, i32 undef, i32 3 + %tmp29 = insertelement <16 x i32> %tmp28, i32 undef, i32 4 + %tmp30 = insertelement <16 x i32> %tmp29, i32 0, i32 5 + %tmp31 = insertelement <16 x i32> %tmp30, i32 undef, i32 6 + %tmp32 = insertelement <16 x i32> %tmp31, i32 undef, i32 7 + %tmp33 = insertelement <16 x i32> %tmp32, i32 undef, i32 8 + %tmp34 = call <4 x float> @llvm.SI.image.sample.c.d.o.v16i32(<16 x i32> %tmp33, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0) + %tmp35 = extractelement <4 x float> %tmp34, i32 0 + %tmp36 = bitcast float %tmp24 to i32 + %tmp37 = insertelement <16 x i32> <i32 212739, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, i32 %tmp36, i32 1 + %tmp38 = insertelement <16 x i32> %tmp37, i32 undef, i32 2 + %tmp39 = insertelement <16 x i32> %tmp38, i32 undef, i32 3 + %tmp40 = insertelement <16 x i32> %tmp39, i32 undef, i32 4 + %tmp41 = insertelement <16 x i32> %tmp40, i32 0, i32 5 + %tmp42 = insertelement <16 x i32> %tmp41, i32 undef, i32 6 + %tmp43 = insertelement <16 x i32> %tmp42, i32 undef, i32 7 + %tmp44 = insertelement <16 x i32> %tmp43, i32 undef, i32 8 + %tmp45 = call <4 x float> @llvm.SI.image.sample.c.d.o.v16i32(<16 x i32> %tmp44, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0) + %tmp46 = extractelement <4 x float> %tmp45, i32 0 + %tmp47 = fmul float %tmp35, %tmp46 + %tmp48 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, float %tmp47, 14 + %tmp49 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp48, float %arg21, 24 + ret <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp49 +} + +declare float @llvm.SI.fs.interp(i32, i32, i32, <2 x i32>) #1 +declare float @llvm.SI.load.const(<16 x i8>, i32) #1 +declare <4 x float> @llvm.SI.image.sample.c.d.o.v16i32(<16 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #1 + +attributes #0 = { "InitialPSInputAddr"="36983" "target-cpu"="tonga" } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/coalescer-subreg-join.mir b/test/CodeGen/AMDGPU/coalescer-subreg-join.mir new file mode 100644 index 000000000000..234fe57b5135 --- /dev/null +++ b/test/CodeGen/AMDGPU/coalescer-subreg-join.mir @@ -0,0 +1,75 @@ +# RUN: llc -march=amdgcn -run-pass simple-register-coalescing -o - %s | FileCheck %s +# Check that %11 and %20 have been coalesced. +# CHECK: IMAGE_SAMPLE_C_D_O_V1_V16 %[[REG:[0-9]+]] +# CHECK: IMAGE_SAMPLE_C_D_O_V1_V16 %[[REG]] + +--- +name: main +alignment: 0 +tracksRegLiveness: true +registers: + - { id: 0, class: sreg_64 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: sreg_256 } + - { id: 4, class: sreg_128 } + - { id: 5, class: sreg_256 } + - { id: 6, class: sreg_128 } + - { id: 7, class: sreg_512 } + - { id: 9, class: vreg_512 } + - { id: 11, class: vreg_512 } + - { id: 18, class: vgpr_32 } + - { id: 20, class: vreg_512 } + - { id: 27, class: vgpr_32 } +liveins: + - { reg: '%sgpr2_sgpr3', virtual-reg: '%0' } + - { reg: '%vgpr2', virtual-reg: '%1' } + - { reg: '%vgpr3', virtual-reg: '%2' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0: + liveins: %sgpr2_sgpr3, %vgpr2, %vgpr3 + + %0 = COPY %sgpr2_sgpr3 + %1 = COPY %vgpr2 + %2 = COPY %vgpr3 + %3 = S_LOAD_DWORDX8_IMM %0, 0, 0 + %4 = S_LOAD_DWORDX4_IMM %0, 12, 0 + %5 = S_LOAD_DWORDX8_IMM %0, 16, 0 + %6 = S_LOAD_DWORDX4_IMM %0, 28, 0 + undef %7.sub0 = S_MOV_B32 212739 + %20 = COPY %7 + %11 = COPY %20 + %11.sub1 = COPY %1 + %11.sub2 = COPY %1 + %11.sub3 = COPY %1 + %11.sub4 = COPY %1 + %11.sub5 = COPY %1 + %11.sub6 = COPY %1 + %11.sub7 = COPY %1 + %11.sub8 = COPY %1 + dead %18 = IMAGE_SAMPLE_C_D_O_V1_V16 %11, %3, %4, 1, 0, 0, 0, 0, 0, 0, -1, implicit %exec + %20.sub1 = COPY %2 + %20.sub2 = COPY %2 + %20.sub3 = COPY %2 + %20.sub4 = COPY %2 + %20.sub5 = COPY %2 + %20.sub6 = COPY %2 + %20.sub7 = COPY %2 + %20.sub8 = COPY %2 + dead %27 = IMAGE_SAMPLE_C_D_O_V1_V16 %20, %5, %6, 1, 0, 0, 0, 0, 0, 0, -1, implicit %exec + +... diff --git a/test/CodeGen/AMDGPU/coalescer_remat.ll b/test/CodeGen/AMDGPU/coalescer_remat.ll index 96730bcf2e8f..4c7875c3a039 100644 --- a/test/CodeGen/AMDGPU/coalescer_remat.ll +++ b/test/CodeGen/AMDGPU/coalescer_remat.ll @@ -12,7 +12,7 @@ declare float @llvm.fma.f32(float, float, float) ; CHECK: v_mov_b32_e32 v{{[0-9]+}}, 0 ; CHECK: v_mov_b32_e32 v{{[0-9]+}}, 0 ; It's probably OK if this is slightly higher: -; CHECK: ; NumVgprs: 9 +; CHECK: ; NumVgprs: 8 define void @foobar(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in, i32 %flag) { entry: %cmpflag = icmp eq i32 %flag, 1 diff --git a/test/CodeGen/AMDGPU/commute-compares.ll b/test/CodeGen/AMDGPU/commute-compares.ll index 731b47cd9ee2..a4c51b233f41 100644 --- a/test/CodeGen/AMDGPU/commute-compares.ll +++ b/test/CodeGen/AMDGPU/commute-compares.ll @@ -7,7 +7,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0 ; -------------------------------------------------------------------------------- ; GCN-LABEL: {{^}}commute_eq_64_i32: -; GCN: v_cmp_eq_i32_e32 vcc, 64, v{{[0-9]+}} +; GCN: v_cmp_eq_u32_e32 vcc, 64, v{{[0-9]+}} define void @commute_eq_64_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i32, i32 addrspace(1)* %in, i32 %tid @@ -20,7 +20,7 @@ define void @commute_eq_64_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 } ; GCN-LABEL: {{^}}commute_ne_64_i32: -; GCN: v_cmp_ne_i32_e32 vcc, 64, v{{[0-9]+}} +; GCN: v_cmp_ne_u32_e32 vcc, 64, v{{[0-9]+}} define void @commute_ne_64_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i32, i32 addrspace(1)* %in, i32 %tid @@ -35,7 +35,7 @@ define void @commute_ne_64_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 ; FIXME: Why isn't this being folded as a constant? ; GCN-LABEL: {{^}}commute_ne_litk_i32: ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3039 -; GCN: v_cmp_ne_i32_e32 vcc, [[K]], v{{[0-9]+}} +; GCN: v_cmp_ne_u32_e32 vcc, [[K]], v{{[0-9]+}} define void @commute_ne_litk_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i32, i32 addrspace(1)* %in, i32 %tid @@ -172,7 +172,7 @@ define void @commute_sle_5_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #1 ; -------------------------------------------------------------------------------- ; GCN-LABEL: {{^}}commute_eq_64_i64: -; GCN: v_cmp_eq_i64_e32 vcc, 64, v{{\[[0-9]+:[0-9]+\]}} +; GCN: v_cmp_eq_u64_e32 vcc, 64, v{{\[[0-9]+:[0-9]+\]}} define void @commute_eq_64_i64(i32 addrspace(1)* %out, i64 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid @@ -185,7 +185,7 @@ define void @commute_eq_64_i64(i32 addrspace(1)* %out, i64 addrspace(1)* %in) #1 } ; GCN-LABEL: {{^}}commute_ne_64_i64: -; GCN: v_cmp_ne_i64_e32 vcc, 64, v{{\[[0-9]+:[0-9]+\]}} +; GCN: v_cmp_ne_u64_e32 vcc, 64, v{{\[[0-9]+:[0-9]+\]}} define void @commute_ne_64_i64(i32 addrspace(1)* %out, i64 addrspace(1)* %in) #1 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 %gep.in = getelementptr i64, i64 addrspace(1)* %in, i32 %tid @@ -693,5 +693,25 @@ define void @commute_uno_2.0_f64(i32 addrspace(1)* %out, double addrspace(1)* %i ret void } + +; FIXME: Should be able to fold this frameindex +; Without commuting the frame index in the pre-regalloc run of +; SIShrinkInstructions, this was using the VOP3 compare. + +; GCN-LABEL: {{^}}commute_frameindex: +; XGCN: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}} + +; GCN: v_mov_b32_e32 [[FI:v[0-9]+]], 0{{$}} +; GCN: v_cmp_eq_u32_e32 vcc, [[FI]], v{{[0-9]+}} +define void @commute_frameindex(i32 addrspace(1)* nocapture %out) #0 { +entry: + %stack0 = alloca i32 + %ptr0 = load volatile i32*, i32* addrspace(1)* undef + %eq = icmp eq i32* %ptr0, %stack0 + %ext = zext i1 %eq to i32 + store volatile i32 %ext, i32 addrspace(1)* %out + ret void +} + attributes #0 = { nounwind readnone } attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/commute_modifiers.ll b/test/CodeGen/AMDGPU/commute_modifiers.ll index bce3fe998c8a..ed4ec82eb3e3 100644 --- a/test/CodeGen/AMDGPU/commute_modifiers.ll +++ b/test/CodeGen/AMDGPU/commute_modifiers.ll @@ -6,7 +6,7 @@ declare float @llvm.fma.f32(float, float, float) nounwind readnone ; FUNC-LABEL: @commute_add_imm_fabs_f32 ; SI: buffer_load_dword [[X:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; SI: v_add_f32_e64 [[REG:v[0-9]+]], 2.0, |[[X]]| +; SI: v_add_f32_e64 [[REG:v[0-9]+]], |[[X]]|, 2.0 ; SI: buffer_store_dword [[REG]] define void @commute_add_imm_fabs_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -20,7 +20,7 @@ define void @commute_add_imm_fabs_f32(float addrspace(1)* %out, float addrspace( ; FUNC-LABEL: @commute_mul_imm_fneg_fabs_f32 ; SI: buffer_load_dword [[X:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; SI: v_mul_f32_e64 [[REG:v[0-9]+]], -4.0, |[[X]]| +; SI: v_mul_f32_e64 [[REG:v[0-9]+]], |[[X]]|, -4.0 ; SI: buffer_store_dword [[REG]] define void @commute_mul_imm_fneg_fabs_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -51,7 +51,7 @@ define void @commute_mul_imm_fneg_f32(float addrspace(1)* %out, float addrspace( ; FUNC-LABEL: @commute_add_lit_fabs_f32 ; SI: buffer_load_dword [[X:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} ; SI: v_mov_b32_e32 [[K:v[0-9]+]], 0x44800000 -; SI: v_add_f32_e64 [[REG:v[0-9]+]], |[[X]]|, [[K]] +; SI: v_add_f32_e64 [[REG:v[0-9]+]], [[K]], |[[X]]| ; SI: buffer_store_dword [[REG]] define void @commute_add_lit_fabs_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 diff --git a/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll b/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll new file mode 100644 index 000000000000..3e167846c22c --- /dev/null +++ b/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll @@ -0,0 +1,144 @@ +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}fold_mi_v_and_0: +; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], 0{{$}} +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] +define void @fold_mi_v_and_0(i32 addrspace(1)* %out) { + %x = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %size = call i32 @llvm.amdgcn.groupstaticsize() + %and = and i32 %size, %x + store i32 %and, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_s_and_0: +; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], 0{{$}} +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] +define void @fold_mi_s_and_0(i32 addrspace(1)* %out, i32 %x) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %and = and i32 %size, %x + store i32 %and, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_v_or_0: +; GCN: v_mbcnt_lo_u32_b32_e64 [[RESULT:v[0-9]+]] +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] +define void @fold_mi_v_or_0(i32 addrspace(1)* %out) { + %x = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %size = call i32 @llvm.amdgcn.groupstaticsize() + %or = or i32 %size, %x + store i32 %or, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_s_or_0: +; GCN: s_load_dword [[SVAL:s[0-9]+]] +; GCN-NOT: [[SVAL]] +; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]] +; GCN-NOT: [[VVAL]] +; GCN: buffer_store_dword [[VVAL]] +define void @fold_mi_s_or_0(i32 addrspace(1)* %out, i32 %x) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %or = or i32 %size, %x + store i32 %or, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_v_xor_0: +; GCN: v_mbcnt_lo_u32_b32_e64 [[RESULT:v[0-9]+]] +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] +define void @fold_mi_v_xor_0(i32 addrspace(1)* %out) { + %x = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0 + %size = call i32 @llvm.amdgcn.groupstaticsize() + %xor = xor i32 %size, %x + store i32 %xor, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_s_xor_0: +; GCN: s_load_dword [[SVAL:s[0-9]+]] +; GCN-NOT: [[SVAL]] +; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]] +; GCN-NOT: [[VVAL]] +; GCN: buffer_store_dword [[VVAL]] +define void @fold_mi_s_xor_0(i32 addrspace(1)* %out, i32 %x) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %xor = xor i32 %size, %x + store i32 %xor, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_s_not_0: +; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], -1{{$}} +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] +define void @fold_mi_s_not_0(i32 addrspace(1)* %out, i32 %x) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %xor = xor i32 %size, -1 + store i32 %xor, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_v_not_0: +; GCN: v_bcnt_u32_b32_e64 v[[RESULT_LO:[0-9]+]], v{{[0-9]+}}, 0{{$}} +; GCN: v_bcnt_u32_b32_e{{[0-9]+}} v[[RESULT_LO:[0-9]+]], v{{[0-9]+}}, v[[RESULT_LO]]{{$}} +; GCN-NEXT: v_not_b32_e32 v[[RESULT_LO]] +; GCN-NEXT: v_mov_b32_e32 v[[RESULT_HI:[0-9]+]], -1{{$}} +; GCN-NEXT: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} +define void @fold_mi_v_not_0(i64 addrspace(1)* %out) { + %vreg = load volatile i64, i64 addrspace(1)* undef + %ctpop = call i64 @llvm.ctpop.i64(i64 %vreg) + %xor = xor i64 %ctpop, -1 + store i64 %xor, i64 addrspace(1)* %out + ret void +} + +; The neg1 appears after folding the not 0 +; GCN-LABEL: {{^}}fold_mi_or_neg1: +; GCN: buffer_load_dwordx2 +; GCN: buffer_load_dwordx2 v{{\[}}[[VREG1_LO:[0-9]+]]:[[VREG1_HI:[0-9]+]]{{\]}} + +; GCN: v_bcnt_u32_b32_e64 v[[RESULT_LO:[0-9]+]], v{{[0-9]+}}, 0{{$}} +; GCN: v_bcnt_u32_b32_e{{[0-9]+}} v[[RESULT_LO:[0-9]+]], v{{[0-9]+}}, v[[RESULT_LO]]{{$}} +; GCN-DAG: v_not_b32_e32 v[[RESULT_LO]], v[[RESULT_LO]] +; GCN-DAG: v_or_b32_e32 v[[RESULT_LO]], v[[VREG1_LO]], v[[RESULT_LO]] +; GCN-DAG: v_mov_b32_e32 v[[RESULT_HI:[0-9]+]], v[[VREG1_HI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} +define void @fold_mi_or_neg1(i64 addrspace(1)* %out) { + %vreg0 = load volatile i64, i64 addrspace(1)* undef + %vreg1 = load volatile i64, i64 addrspace(1)* undef + %ctpop = call i64 @llvm.ctpop.i64(i64 %vreg0) + %xor = xor i64 %ctpop, -1 + %or = or i64 %xor, %vreg1 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fold_mi_and_neg1: +; GCN: v_bcnt_u32_b32 +; GCN: v_bcnt_u32_b32 +; GCN: v_not_b32 +; GCN: v_and_b32 +; GCN-NOT: v_and_b32 +define void @fold_mi_and_neg1(i64 addrspace(1)* %out) { + %vreg0 = load volatile i64, i64 addrspace(1)* undef + %vreg1 = load volatile i64, i64 addrspace(1)* undef + %ctpop = call i64 @llvm.ctpop.i64(i64 %vreg0) + %xor = xor i64 %ctpop, -1 + %and = and i64 %xor, %vreg1 + store i64 %and, i64 addrspace(1)* %out + ret void +} + +declare i64 @llvm.ctpop.i64(i64) #1 +declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1 +declare i32 @llvm.amdgcn.groupstaticsize() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll new file mode 100644 index 000000000000..13383cbc1741 --- /dev/null +++ b/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll @@ -0,0 +1,296 @@ +; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -check-prefix=VMEM -check-prefix=GCN %s +; RUN: llc -O0 -mtriple=amdgcn--amdhsa -march=amdgcn -amdgpu-spill-sgpr-to-vgpr=1 -verify-machineinstrs < %s | FileCheck -check-prefix=VGPR -check-prefix=GCN %s + +; Verify registers used for tracking exec mask changes when all +; registers are spilled at the end of the block. The SGPR spill +; placement relative to the exec modifications are important. + +; FIXME: This checks with SGPR to VGPR spilling disabled, but this may +; not work correctly in cases where no workitems take a branch. + + +; GCN-LABEL: {{^}}divergent_if_endif: + +; GCN: {{^}}; BB#0: +; GCN: s_mov_b32 m0, -1 +; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] + +; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 +; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec +; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, [[CMP0]] +; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}} + +; Spill saved exec +; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] +; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] + + +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 ; 4-byte Folded Spill +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:4 ; 4-byte Folded Spill + +; Spill load +; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} + +; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN: mask branch [[ENDIF:BB[0-9]+_[0-9]+]] + +; GCN: {{^}}BB{{[0-9]+}}_1: ; %if +; GCN: s_mov_b32 m0, -1 +; GCN: ds_read_b32 [[LOAD1:v[0-9]+]] +; GCN: buffer_load_dword [[RELOAD_LOAD0:v[0-9]+]], off, s[0:3], s7 offset:[[LOAD0_OFFSET]] ; 4-byte Folded Reload +; GCN: s_waitcnt vmcnt(0) + +; Spill val register +; GCN: v_add_i32_e32 [[VAL:v[0-9]+]], vcc, [[LOAD1]], [[RELOAD_LOAD0]] +; GCN: buffer_store_dword [[VAL]], off, s[0:3], s7 offset:[[VAL_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; GCN: s_waitcnt vmcnt(0) + +; VMEM: [[ENDIF]]: +; Reload and restore exec mask +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] + + + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:4 ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] + +; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} + +; Restore val +; GCN: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], s7 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload + +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RELOAD_VAL]] +define void @divergent_if_endif(i32 addrspace(1)* %out) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %load0 = load volatile i32, i32 addrspace(3)* undef + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %if, label %endif + +if: + %load1 = load volatile i32, i32 addrspace(3)* undef + %val = add i32 %load0, %load1 + br label %endif + +endif: + %tmp4 = phi i32 [ %val, %if ], [ 0, %entry ] + store i32 %tmp4, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}divergent_loop: +; GCN: {{^}}; BB#0: + +; GCN: s_mov_b32 m0, -1 +; GCN: ds_read_b32 [[LOAD0:v[0-9]+]] + +; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], s{{[0-9]+}}, v0 + +; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec +; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] +; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}} + +; Spill load +; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 ; 4-byte Folded Spill + +; Spill saved exec +; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] +; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] + + +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 offset:16 ; 4-byte Folded Spill +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:20 ; 4-byte Folded Spill + +; GCN: s_mov_b64 exec, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}} + +; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN-NEXT: ; mask branch [[END:BB[0-9]+_[0-9]+]] +; GCN-NEXT: s_cbranch_execz [[END]] + + +; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: +; GCN: buffer_load_dword v[[VAL_LOOP_RELOAD:[0-9]+]], off, s[0:3], s7 ; 4-byte Folded Reload +; GCN: v_subrev_i32_e32 [[VAL_LOOP:v[0-9]+]], vcc, v{{[0-9]+}}, v[[VAL_LOOP_RELOAD]] +; GCN: v_cmp_ne_u32_e32 vcc, +; GCN: s_and_b64 vcc, exec, vcc +; GCN: buffer_store_dword [[VAL_LOOP]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN-NEXT: s_cbranch_vccnz [[LOOP]] + + +; GCN: [[END]]: +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:16 ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:20 ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] + +; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} +; GCN: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], s7 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload + +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[VAL_END]] +define void @divergent_loop(i32 addrspace(1)* %out) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %load0 = load volatile i32, i32 addrspace(3)* undef + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %loop, label %end + +loop: + %i = phi i32 [ %i.inc, %loop ], [ 0, %entry ] + %val = phi i32 [ %val.sub, %loop ], [ %load0, %entry ] + %load1 = load volatile i32, i32 addrspace(3)* undef + %i.inc = add i32 %i, 1 + %val.sub = sub i32 %val, %load1 + %cmp1 = icmp ne i32 %i, 256 + br i1 %cmp1, label %loop, label %end + +end: + %tmp4 = phi i32 [ %val.sub, %loop ], [ 0, %entry ] + store i32 %tmp4, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}divergent_if_else_endif: +; GCN: {{^}}; BB#0: + +; GCN: s_mov_b32 m0, -1 +; VMEM: ds_read_b32 [[LOAD0:v[0-9]+]] + +; GCN: v_cmp_ne_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], v0, + +; GCN: s_mov_b64 s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, exec +; GCN: s_and_b64 s{{\[}}[[ANDEXEC_LO:[0-9]+]]:[[ANDEXEC_HI:[0-9]+]]{{\]}}, s{{\[}}[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]{{\]}}, [[CMP0]] +; GCN: s_xor_b64 s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}}, s{{\[}}[[ANDEXEC_LO]]:[[ANDEXEC_HI]]{{\]}}, s{{\[}}[[SAVEEXEC_LO]]:[[SAVEEXEC_HI]]{{\]}} + +; Spill load +; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], s7 ; 4-byte Folded Spill + +; Spill saved exec +; VGPR: v_writelane_b32 [[SPILL_VGPR:v[0-9]+]], s[[SAVEEXEC_LO]], [[SAVEEXEC_LO_LANE:[0-9]+]] +; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[SAVEEXEC_HI]], [[SAVEEXEC_HI_LANE:[0-9]+]] + +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_LO:[0-9]+]], s[[SAVEEXEC_LO]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_LO]], off, s[0:3], s7 offset:[[SAVEEXEC_LO_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; VMEM: v_mov_b32_e32 v[[V_SAVEEXEC_HI:[0-9]+]], s[[SAVEEXEC_HI]] +; VMEM: buffer_store_dword v[[V_SAVEEXEC_HI]], off, s[0:3], s7 offset:[[SAVEEXEC_HI_OFFSET:[0-9]+]] ; 4-byte Folded Spill + +; GCN: s_mov_b64 exec, [[CMP0]] +; GCN: s_waitcnt vmcnt(0) expcnt(0) + +; FIXME: It makes no sense to put this skip here +; GCN-NEXT: ; mask branch [[FLOW:BB[0-9]+_[0-9]+]] +; GCN: s_cbranch_execz [[FLOW]] +; GCN-NEXT: s_branch [[ELSE:BB[0-9]+_[0-9]+]] + +; GCN: [[FLOW]]: ; %Flow +; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]] +; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]] + + +; VMEM: buffer_load_dword v[[FLOW_V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:[[SAVEEXEC_LO_OFFSET]] +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC_LO]] + +; VMEM: buffer_load_dword v[[FLOW_V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:[[SAVEEXEC_HI_OFFSET]] ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[FLOW_V_RELOAD_SAVEEXEC_HI]] + +; GCN: s_or_saveexec_b64 s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}}, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} + +; Regular spill value restored after exec modification +; GCN: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], s7 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload + + +; Spill saved exec +; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]], [[FLOW_SAVEEXEC_LO_LANE:[0-9]+]] +; VGPR: v_writelane_b32 [[SPILL_VGPR]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]], [[FLOW_SAVEEXEC_HI_LANE:[0-9]+]] + + +; VMEM: v_mov_b32_e32 v[[FLOW_V_SAVEEXEC_LO:[0-9]+]], s[[FLOW_S_RELOAD_SAVEEXEC_LO]] +; VMEM: buffer_store_dword v[[FLOW_V_SAVEEXEC_LO]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_LO_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; VMEM: v_mov_b32_e32 v[[FLOW_V_SAVEEXEC_HI:[0-9]+]], s[[FLOW_S_RELOAD_SAVEEXEC_HI]] +; VMEM: buffer_store_dword v[[FLOW_V_SAVEEXEC_HI]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_HI_OFFSET:[0-9]+]] ; 4-byte Folded Spill + +; GCN: buffer_store_dword [[FLOW_VAL]], off, s[0:3], s7 offset:[[RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; GCN: s_xor_b64 exec, exec, s{{\[}}[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]{{\]}} +; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) +; GCN-NEXT: ; mask branch [[ENDIF:BB[0-9]+_[0-9]+]] +; GCN-NEXT: s_cbranch_execz [[ENDIF]] + + +; GCN: BB{{[0-9]+}}_2: ; %if +; GCN: ds_read_b32 +; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 ; 4-byte Folded Reload +; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] +; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[RESULT_OFFSET]] ; 4-byte Folded Spill +; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN-NEXT: s_branch [[ENDIF:BB[0-9]+_[0-9]+]] + +; GCN: [[ELSE]]: ; %else +; GCN: buffer_load_dword v[[LOAD0_RELOAD:[0-9]+]], off, s[0:3], s7 ; 4-byte Folded Reload +; GCN: v_subrev_i32_e32 [[SUB:v[0-9]+]], vcc, v{{[0-9]+}}, v[[LOAD0_RELOAD]] +; GCN: buffer_store_dword [[ADD]], off, s[0:3], s7 offset:[[FLOW_RESULT_OFFSET:[0-9]+]] ; 4-byte Folded Spill +; GCN: s_waitcnt vmcnt(0) expcnt(0) +; GCN-NEXT: s_branch [[FLOW]] + +; GCN: [[ENDIF]]: +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_LO_LANE]] +; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_HI_LANE]] + + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_LO:[0-9]+]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_LO_OFFSET]] ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], v[[V_RELOAD_SAVEEXEC_LO]] + +; VMEM: buffer_load_dword v[[V_RELOAD_SAVEEXEC_HI:[0-9]+]], off, s[0:3], s7 offset:[[FLOW_SAVEEXEC_HI_OFFSET]] ; 4-byte Folded Reload +; VMEM: s_waitcnt vmcnt(0) +; VMEM: v_readfirstlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], v[[V_RELOAD_SAVEEXEC_HI]] + +; GCN: s_or_b64 exec, exec, s{{\[}}[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]{{\]}} + +; GCN: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], s7 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload +; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RESULT]] +define void @divergent_if_else_endif(i32 addrspace(1)* %out) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %load0 = load volatile i32, i32 addrspace(3)* undef + %cmp0 = icmp eq i32 %tid, 0 + br i1 %cmp0, label %if, label %else + +if: + %load1 = load volatile i32, i32 addrspace(3)* undef + %val0 = add i32 %load0, %load1 + br label %endif + +else: + %load2 = load volatile i32, i32 addrspace(3)* undef + %val1 = sub i32 %load0, %load2 + br label %endif + +endif: + %result = phi i32 [ %val0, %if ], [ %val1, %else ] + store i32 %result, i32 addrspace(1)* %out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/convergent-inlineasm.ll b/test/CodeGen/AMDGPU/convergent-inlineasm.ll index 55a38e576ad1..755f439c6863 100644 --- a/test/CodeGen/AMDGPU/convergent-inlineasm.ll +++ b/test/CodeGen/AMDGPU/convergent-inlineasm.ll @@ -3,12 +3,13 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0 ; GCN-LABEL: {{^}}convergent_inlineasm: ; GCN: BB#0: -; GCN: v_cmp_ne_i32_e64 -; GCN: BB#1: +; GCN: v_cmp_ne_u32_e64 +; GCN: ; mask branch +; GCN: BB{{[0-9]+_[0-9]+}}: define void @convergent_inlineasm(i64 addrspace(1)* nocapture %arg) { bb: %tmp = call i32 @llvm.amdgcn.workitem.id.x() - %tmp1 = tail call i64 asm "v_cmp_ne_i32_e64 $0, 0, $1", "=s,v"(i32 1) #1 + %tmp1 = tail call i64 asm "v_cmp_ne_u32_e64 $0, 0, $1", "=s,v"(i32 1) #1 %tmp2 = icmp eq i32 %tmp, 8 br i1 %tmp2, label %bb3, label %bb5 @@ -22,13 +23,16 @@ bb5: ; preds = %bb3, %bb } ; GCN-LABEL: {{^}}nonconvergent_inlineasm: -; GCN: BB#1: -; GCN: v_cmp_ne_i32_e64 -; GCN: BB1_2: +; GCN: ; mask branch + +; GCN: BB{{[0-9]+_[0-9]+}}: +; GCN: v_cmp_ne_u32_e64 + +; GCN: BB{{[0-9]+_[0-9]+}}: define void @nonconvergent_inlineasm(i64 addrspace(1)* nocapture %arg) { bb: %tmp = call i32 @llvm.amdgcn.workitem.id.x() - %tmp1 = tail call i64 asm "v_cmp_ne_i32_e64 $0, 0, $1", "=s,v"(i32 1) + %tmp1 = tail call i64 asm "v_cmp_ne_u32_e64 $0, 0, $1", "=s,v"(i32 1) %tmp2 = icmp eq i32 %tmp, 8 br i1 %tmp2, label %bb3, label %bb5 diff --git a/test/CodeGen/AMDGPU/copy-illegal-type.ll b/test/CodeGen/AMDGPU/copy-illegal-type.ll index 00d2257f4adc..6918dff74d57 100644 --- a/test/CodeGen/AMDGPU/copy-illegal-type.ll +++ b/test/CodeGen/AMDGPU/copy-illegal-type.ll @@ -1,10 +1,13 @@ -; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s + +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone +declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone ; FUNC-LABEL: {{^}}test_copy_v4i8: -; SI: buffer_load_dword [[REG:v[0-9]+]] -; SI: buffer_store_dword [[REG]] -; SI: s_endpgm +; GCN: buffer_load_dword [[REG:v[0-9]+]] +; GCN: buffer_store_dword [[REG]] +; GCN: s_endpgm define void @test_copy_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store <4 x i8> %val, <4 x i8> addrspace(1)* %out, align 4 @@ -12,10 +15,10 @@ define void @test_copy_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* } ; FUNC-LABEL: {{^}}test_copy_v4i8_x2: -; SI: buffer_load_dword [[REG:v[0-9]+]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: s_endpgm +; GCN: buffer_load_dword [[REG:v[0-9]+]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: s_endpgm define void @test_copy_v4i8_x2(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store <4 x i8> %val, <4 x i8> addrspace(1)* %out0, align 4 @@ -24,11 +27,11 @@ define void @test_copy_v4i8_x2(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace( } ; FUNC-LABEL: {{^}}test_copy_v4i8_x3: -; SI: buffer_load_dword [[REG:v[0-9]+]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: s_endpgm +; GCN: buffer_load_dword [[REG:v[0-9]+]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: s_endpgm define void @test_copy_v4i8_x3(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store <4 x i8> %val, <4 x i8> addrspace(1)* %out0, align 4 @@ -38,12 +41,12 @@ define void @test_copy_v4i8_x3(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace( } ; FUNC-LABEL: {{^}}test_copy_v4i8_x4: -; SI: buffer_load_dword [[REG:v[0-9]+]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: buffer_store_dword [[REG]] -; SI: s_endpgm +; GCN: buffer_load_dword [[REG:v[0-9]+]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: buffer_store_dword [[REG]] +; GCN: s_endpgm define void @test_copy_v4i8_x4(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %out3, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store <4 x i8> %val, <4 x i8> addrspace(1)* %out0, align 4 @@ -54,14 +57,14 @@ define void @test_copy_v4i8_x4(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace( } ; FUNC-LABEL: {{^}}test_copy_v4i8_extra_use: -; SI: buffer_load_dword -; SI-DAG: v_lshrrev_b32 -; SI: v_and_b32 -; SI: v_or_b32 -; SI-DAG: buffer_store_dword -; SI-DAG: buffer_store_dword +; GCN: buffer_load_dword +; GCN-DAG: v_lshrrev_b32 +; GCN: v_and_b32 +; GCN: v_or_b32 +; GCN-DAG: buffer_store_dword +; GCN-DAG: buffer_store_dword -; SI: s_endpgm +; GCN: s_endpgm define void @test_copy_v4i8_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 %add = add <4 x i8> %val, <i8 9, i8 9, i8 9, i8 9> @@ -70,18 +73,22 @@ define void @test_copy_v4i8_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> add ret void } +; FIXME: Need to handle non-uniform case for function below (load without gep). ; FUNC-LABEL: {{^}}test_copy_v4i8_x2_extra_use: -; SI: buffer_load_dword -; SI-DAG: v_lshrrev_b32 +; GCN: {{buffer|flat}}_load_dword +; GCN-DAG: v_lshrrev_b32 ; SI-DAG: v_add_i32 -; SI-DAG: v_and_b32 -; SI-DAG: v_or_b32 -; SI-DAG: buffer_store_dword -; SI: buffer_store_dword -; SI: buffer_store_dword -; SI: s_endpgm +; VI-DAG: v_add_u16 +; GCN-DAG: v_and_b32 +; GCN-DAG: v_or_b32 +; GCN-DAG: {{buffer|flat}}_store_dword +; GCN: {{buffer|flat}}_store_dword +; GCN: {{buffer|flat}}_store_dword +; GCN: s_endpgm define void @test_copy_v4i8_x2_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> addrspace(1)* %out1, <4 x i8> addrspace(1)* %out2, <4 x i8> addrspace(1)* %in) nounwind { - %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %in.ptr = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x + %val = load <4 x i8>, <4 x i8> addrspace(1)* %in.ptr, align 4 %add = add <4 x i8> %val, <i8 9, i8 9, i8 9, i8 9> store <4 x i8> %val, <4 x i8> addrspace(1)* %out0, align 4 store <4 x i8> %add, <4 x i8> addrspace(1)* %out1, align 4 @@ -90,10 +97,10 @@ define void @test_copy_v4i8_x2_extra_use(<4 x i8> addrspace(1)* %out0, <4 x i8> } ; FUNC-LABEL: {{^}}test_copy_v3i8_align4: -; SI: buffer_load_dword -; SI-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} -; SI-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} -; SI: s_endpgm +; GCN: buffer_load_dword +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} +; GCN: s_endpgm define void @test_copy_v3i8_align4(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind { %val = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 4 store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 4 @@ -101,11 +108,11 @@ define void @test_copy_v3i8_align4(<3 x i8> addrspace(1)* %out, <3 x i8> addrspa } ; FUNC-LABEL: {{^}}test_copy_v3i8_align2: -; SI-DAG: buffer_load_ushort v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} -; SI-DAG: buffer_load_ubyte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} -; SI-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} -; SI-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} -; SI: s_endpgm +; GCN-DAG: buffer_load_ushort v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; GCN-DAG: buffer_load_ubyte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:2{{$}} +; GCN: s_endpgm define void @test_copy_v3i8_align2(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind { %val = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 2 store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 2 @@ -113,14 +120,14 @@ define void @test_copy_v3i8_align2(<3 x i8> addrspace(1)* %out, <3 x i8> addrspa } ; FUNC-LABEL: {{^}}test_copy_v3i8_align1: -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: s_endpgm +; GCN: buffer_store_byte +; GCN: buffer_store_byte +; GCN: buffer_store_byte +; GCN: s_endpgm define void @test_copy_v3i8_align1(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) nounwind { %val = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 1 store <3 x i8> %val, <3 x i8> addrspace(1)* %out, align 1 @@ -128,12 +135,12 @@ define void @test_copy_v3i8_align1(<3 x i8> addrspace(1)* %out, <3 x i8> addrspa } ; FUNC-LABEL: {{^}}test_copy_v4i8_volatile_load: -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_store_dword -; SI: s_endpgm +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_store_dword +; GCN: s_endpgm define void @test_copy_v4i8_volatile_load(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind { %val = load volatile <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store <4 x i8> %val, <4 x i8> addrspace(1)* %out, align 4 @@ -141,15 +148,15 @@ define void @test_copy_v4i8_volatile_load(<4 x i8> addrspace(1)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}test_copy_v4i8_volatile_store: -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_load_ubyte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: buffer_store_byte -; SI: s_endpgm +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_load_ubyte +; GCN: buffer_store_byte +; GCN: buffer_store_byte +; GCN: buffer_store_byte +; GCN: buffer_store_byte +; GCN: s_endpgm define void @test_copy_v4i8_volatile_store(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind { %val = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 store volatile <4 x i8> %val, <4 x i8> addrspace(1)* %out, align 4 diff --git a/test/CodeGen/AMDGPU/ctlz.ll b/test/CodeGen/AMDGPU/ctlz.ll index 6d2d260177e5..13f38d25aa8b 100644 --- a/test/CodeGen/AMDGPU/ctlz.ll +++ b/test/CodeGen/AMDGPU/ctlz.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s declare i7 @llvm.ctlz.i7(i7, i1) nounwind readnone @@ -17,13 +17,13 @@ declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1) nounwind readnone declare i32 @llvm.r600.read.tidig.x() nounwind readnone ; FUNC-LABEL: {{^}}s_ctlz_i32: -; SI: s_load_dword [[VAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} -; SI-DAG: s_flbit_i32_b32 [[CTLZ:s[0-9]+]], [[VAL]] -; SI-DAG: v_cmp_eq_i32_e64 [[CMPZ:s\[[0-9]+:[0-9]+\]]], 0, [[VAL]] -; SI-DAG: v_mov_b32_e32 [[VCTLZ:v[0-9]+]], [[CTLZ]] -; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], [[VCTLZ]], 32, [[CMPZ]] -; SI: buffer_store_dword [[RESULT]] -; SI: s_endpgm +; GCN: s_load_dword [[VAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; GCN-DAG: s_flbit_i32_b32 [[CTLZ:s[0-9]+]], [[VAL]] +; GCN-DAG: v_cmp_ne_u32_e64 vcc, [[VAL]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 [[VCTLZ:v[0-9]+]], [[CTLZ]] +; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], 32, [[VCTLZ]], vcc +; GCN: buffer_store_dword [[RESULT]] +; GCN: s_endpgm ; EG: FFBH_UINT ; EG: CNDE_INT @@ -34,12 +34,12 @@ define void @s_ctlz_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind { } ; FUNC-LABEL: {{^}}v_ctlz_i32: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI-DAG: v_ffbh_u32_e32 [[CTLZ:v[0-9]+]], [[VAL]] -; SI-DAG: v_cmp_eq_i32_e32 vcc, 0, [[CTLZ]] -; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], [[CTLZ]], 32, vcc -; SI: buffer_store_dword [[RESULT]], -; SI: s_endpgm +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN-DAG: v_ffbh_u32_e32 [[CTLZ:v[0-9]+]], [[VAL]] +; GCN-DAG: v_cmp_ne_u32_e32 vcc, 0, [[CTLZ]] +; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], 32, [[CTLZ]], vcc +; GCN: buffer_store_dword [[RESULT]], +; GCN: s_endpgm ; EG: FFBH_UINT ; EG: CNDE_INT @@ -51,11 +51,11 @@ define void @v_ctlz_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalia } ; FUNC-LABEL: {{^}}v_ctlz_v2i32: -; SI: buffer_load_dwordx2 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: buffer_store_dwordx2 -; SI: s_endpgm +; GCN: buffer_load_dwordx2 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: buffer_store_dwordx2 +; GCN: s_endpgm ; EG: FFBH_UINT ; EG: CNDE_INT @@ -69,13 +69,13 @@ define void @v_ctlz_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x i32> addrsp } ; FUNC-LABEL: {{^}}v_ctlz_v4i32: -; SI: buffer_load_dwordx4 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: buffer_store_dwordx4 -; SI: s_endpgm +; GCN: buffer_load_dwordx4 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: buffer_store_dwordx4 +; GCN: s_endpgm ; EG-DAG: FFBH_UINT @@ -97,12 +97,10 @@ define void @v_ctlz_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrsp } ; FUNC-LABEL: {{^}}v_ctlz_i8: -; SI: buffer_load_ubyte [[VAL:v[0-9]+]], -; SI-DAG: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] -; SI-DAG: v_cmp_eq_i32_e32 vcc, 0, [[CTLZ]] -; SI-DAG: v_cndmask_b32_e64 [[CORRECTED_FFBH:v[0-9]+]], [[FFBH]], 32, vcc -; SI: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, 0xffffffe8, [[CORRECTED_FFBH]] -; SI: buffer_store_byte [[RESULT]], +; GCN: buffer_load_ubyte [[VAL:v[0-9]+]], +; GCN-DAG: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_byte [[RESULT]], +; GCN: s_endpgm define void @v_ctlz_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind { %val = load i8, i8 addrspace(1)* %valptr %ctlz = call i8 @llvm.ctlz.i8(i8 %val, i1 false) nounwind readnone @@ -111,16 +109,16 @@ define void @v_ctlz_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias % } ; FUNC-LABEL: {{^}}s_ctlz_i64: -; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} -; SI-DAG: v_cmp_eq_i32_e64 vcc, 0, s[[HI]] -; SI-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]] -; SI-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32 -; SI-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]] -; SI-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[ADD]] -; SI-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]] -; SI-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]] -; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} -; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} +; GCN: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; GCN-DAG: v_cmp_eq_u32_e64 vcc, s[[HI]], 0{{$}} +; GCN-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]] +; GCN-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32 +; GCN-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]] +; GCN-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[ADD]] +; GCN-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]] +; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]] +; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} +; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} define void @s_ctlz_i64(i64 addrspace(1)* noalias %out, i64 %val) nounwind { %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 false) store i64 %ctlz, i64 addrspace(1)* %out @@ -136,17 +134,17 @@ define void @s_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64 %val) nounwind } ; FUNC-LABEL: {{^}}v_ctlz_i64: -; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} -; SI-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}} -; SI-DAG: v_cmp_eq_i32_e64 [[CMPHI:s\[[0-9]+:[0-9]+\]]], 0, v[[HI]] -; SI-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]] -; SI-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]] -; SI-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]] -; SI-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[ADD]], [[CMPHI]] -; SI-DAG: v_or_b32_e32 [[OR:v[0-9]+]], v[[LO]], v[[HI]] -; SI-DAG: v_cmp_eq_i32_e32 vcc, 0, [[OR]] -; SI-DAG: v_cndmask_b32_e64 v[[CLTZ_LO:[0-9]+]], v[[CTLZ:[0-9]+]], 64, vcc -; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CLTZ_LO]]:[[CTLZ_HI]]{{\]}} +; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} +; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}} +; GCN-DAG: v_cmp_eq_u32_e64 [[CMPHI:s\[[0-9]+:[0-9]+\]]], 0, v[[HI]] +; GCN-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]] +; GCN-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]] +; GCN-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]] +; GCN-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[ADD]], [[CMPHI]] +; GCN-DAG: v_or_b32_e32 [[OR:v[0-9]+]], v[[HI]], v[[LO]] +; GCN-DAG: v_cmp_ne_u32_e32 vcc, 0, [[OR]] +; GCN-DAG: v_cndmask_b32_e32 v[[CLTZ_LO:[0-9]+]], 64, v[[CTLZ:[0-9]+]], vcc +; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CLTZ_LO]]:[[CTLZ_HI]]{{\]}} define void @v_ctlz_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind { %tid = call i32 @llvm.r600.read.tidig.x() %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid @@ -170,10 +168,10 @@ define void @v_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64 addrspace(1)* } ; FUNC-LABEL: {{^}}v_ctlz_i32_sel_eq_neg1: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], -; SI: s_endpgm +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], +; GCN: s_endpgm define void @v_ctlz_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 false) nounwind readnone @@ -184,10 +182,10 @@ define void @v_ctlz_i64_trunc(i32 addrspace(1)* noalias %out, i64 addrspace(1)* } ; FUNC-LABEL: {{^}}v_ctlz_i32_sel_ne_neg1: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], -; SI: s_endpgm +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], +; GCN: s_endpgm define void @v_ctlz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 false) nounwind readnone @@ -199,11 +197,11 @@ define void @v_ctlz_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspac ; TODO: Should be able to eliminate select here as well. ; FUNC-LABEL: {{^}}v_ctlz_i32_sel_eq_bitwidth: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: s_endpgm +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: s_endpgm define void @v_ctlz_i32_sel_eq_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 false) nounwind readnone @@ -214,11 +212,11 @@ define void @v_ctlz_i32_sel_eq_bitwidth(i32 addrspace(1)* noalias %out, i32 addr } ; FUNC-LABEL: {{^}}v_ctlz_i32_sel_ne_bitwidth: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: s_endpgm +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: s_endpgm define void @v_ctlz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 false) nounwind readnone @@ -229,11 +227,13 @@ define void @v_ctlz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addr } ; FUNC-LABEL: {{^}}v_ctlz_i8_sel_eq_neg1: -; SI: buffer_load_ubyte [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] -; SI: buffer_store_byte [[FFBH]], +; GCN: {{buffer|flat}}_load_ubyte [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] +; GCN: {{buffer|flat}}_store_byte [[FFBH]], define void @v_ctlz_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind { - %val = load i8, i8 addrspace(1)* %valptr + %tid = call i32 @llvm.r600.read.tidig.x() + %valptr.gep = getelementptr i8, i8 addrspace(1)* %valptr, i32 %tid + %val = load i8, i8 addrspace(1)* %valptr.gep %ctlz = call i8 @llvm.ctlz.i8(i8 %val, i1 false) nounwind readnone %cmp = icmp eq i8 %val, 0 %sel = select i1 %cmp, i8 -1, i8 %ctlz @@ -254,13 +254,16 @@ define void @v_ctlz_i32_sel_ne_bitwidth(i32 addrspace(1)* noalias %out, i32 addr ret void } +; FIXME: Need to handle non-uniform case for function below (load without gep). ; FUNC-LABEL: {{^}}v_ctlz_i7_sel_eq_neg1: -; SI: buffer_load_ubyte [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] -; SI: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0x7f, [[FFBH]] -; SI: buffer_store_byte [[TRUNC]], - define void @v_ctlz_i7_sel_eq_neg1(i7 addrspace(1)* noalias %out, i7 addrspace(1)* noalias %valptr) nounwind { - %val = load i7, i7 addrspace(1)* %valptr +; GCN: {{buffer|flat}}_load_ubyte [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] +; GCN: v_and_b32_e32 [[TRUNC:v[0-9]+]], 0x7f, [[FFBH]] +; GCN: {{buffer|flat}}_store_byte [[TRUNC]], +define void @v_ctlz_i7_sel_eq_neg1(i7 addrspace(1)* noalias %out, i7 addrspace(1)* noalias %valptr) nounwind { + %tid = call i32 @llvm.r600.read.tidig.x() + %valptr.gep = getelementptr i7, i7 addrspace(1)* %valptr, i32 %tid + %val = load i7, i7 addrspace(1)* %valptr.gep %ctlz = call i7 @llvm.ctlz.i7(i7 %val, i1 false) nounwind readnone %cmp = icmp eq i7 %val, 0 %sel = select i1 %cmp, i7 -1, i7 %ctlz diff --git a/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/test/CodeGen/AMDGPU/ctlz_zero_undef.ll index 65e8205317b6..900938bab700 100644 --- a/test/CodeGen/AMDGPU/ctlz_zero_undef.ll +++ b/test/CodeGen/AMDGPU/ctlz_zero_undef.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC -check-prefix=GCN %s ; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s declare i8 @llvm.ctlz.i8(i8, i1) nounwind readnone @@ -15,11 +15,11 @@ declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1) nounwind readnone declare i32 @llvm.r600.read.tidig.x() nounwind readnone ; FUNC-LABEL: {{^}}s_ctlz_zero_undef_i32: -; SI: s_load_dword [[VAL:s[0-9]+]], -; SI: s_flbit_i32_b32 [[SRESULT:s[0-9]+]], [[VAL]] -; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] -; SI: buffer_store_dword [[VRESULT]], -; SI: s_endpgm +; GCN: s_load_dword [[VAL:s[0-9]+]], +; GCN: s_flbit_i32_b32 [[SRESULT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] +; GCN: buffer_store_dword [[VRESULT]], +; GCN: s_endpgm ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] ; EG: FFBH_UINT {{\*? *}}[[RESULT]] define void @s_ctlz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 %val) nounwind { @@ -29,10 +29,10 @@ define void @s_ctlz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 %val) nou } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], -; SI: s_endpgm +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], +; GCN: s_endpgm ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+\.[XYZW]]] ; EG: FFBH_UINT {{\*? *}}[[RESULT]] define void @v_ctlz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { @@ -43,11 +43,11 @@ define void @v_ctlz_zero_undef_i32(i32 addrspace(1)* noalias %out, i32 addrspace } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_v2i32: -; SI: buffer_load_dwordx2 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: buffer_store_dwordx2 -; SI: s_endpgm +; GCN: buffer_load_dwordx2 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: buffer_store_dwordx2 +; GCN: s_endpgm ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+]]{{\.[XYZW]}} ; EG: FFBH_UINT {{\*? *}}[[RESULT]] ; EG: FFBH_UINT {{\*? *}}[[RESULT]] @@ -59,13 +59,13 @@ define void @v_ctlz_zero_undef_v2i32(<2 x i32> addrspace(1)* noalias %out, <2 x } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_v4i32: -; SI: buffer_load_dwordx4 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: v_ffbh_u32_e32 -; SI: buffer_store_dwordx4 -; SI: s_endpgm +; GCN: buffer_load_dwordx4 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: v_ffbh_u32_e32 +; GCN: buffer_store_dwordx4 +; GCN: s_endpgm ; EG: MEM_RAT_CACHELESS STORE_RAW [[RESULT:T[0-9]+]]{{\.[XYZW]}} ; EG: FFBH_UINT {{\*? *}}[[RESULT]] ; EG: FFBH_UINT {{\*? *}}[[RESULT]] @@ -79,10 +79,9 @@ define void @v_ctlz_zero_undef_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i8: -; SI: buffer_load_ubyte [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] -; SI: v_add_i32_e32 [[RESULT:v[0-9]+]], vcc, 0xffffffe8, [[FFBH]] -; SI: buffer_store_byte [[RESULT]], +; GCN: buffer_load_ubyte [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_byte [[RESULT]], define void @v_ctlz_zero_undef_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind { %val = load i8, i8 addrspace(1)* %valptr %ctlz = call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone @@ -91,16 +90,16 @@ define void @v_ctlz_zero_undef_i8(i8 addrspace(1)* noalias %out, i8 addrspace(1) } ; FUNC-LABEL: {{^}}s_ctlz_zero_undef_i64: -; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} -; SI-DAG: v_cmp_eq_i32_e64 vcc, 0, s[[HI]] -; SI-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]] -; SI-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32 -; SI-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]] -; SI-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[FFBH_LO]] -; SI-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]] -; SI-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]] -; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} -; SI: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} +; GCN: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; GCN-DAG: v_cmp_eq_u32_e64 vcc, s[[HI]], 0{{$}} +; GCN-DAG: s_flbit_i32_b32 [[FFBH_LO:s[0-9]+]], s[[LO]] +; GCN-DAG: s_add_i32 [[ADD:s[0-9]+]], [[FFBH_LO]], 32 +; GCN-DAG: s_flbit_i32_b32 [[FFBH_HI:s[0-9]+]], s[[HI]] +; GCN-DAG: v_mov_b32_e32 [[VFFBH_LO:v[0-9]+]], [[FFBH_LO]] +; GCN-DAG: v_mov_b32_e32 [[VFFBH_HI:v[0-9]+]], [[FFBH_HI]] +; GCN-DAG: v_cndmask_b32_e32 v[[CTLZ:[0-9]+]], [[VFFBH_HI]], [[VFFBH_LO]] +; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} +; GCN: {{buffer|flat}}_store_dwordx2 v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} define void @s_ctlz_zero_undef_i64(i64 addrspace(1)* noalias %out, i64 %val) nounwind { %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true) store i64 %ctlz, i64 addrspace(1)* %out @@ -116,14 +115,14 @@ define void @s_ctlz_zero_undef_i64_trunc(i32 addrspace(1)* noalias %out, i64 %va } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i64: -; SI-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}} -; SI-DAG: v_cmp_eq_i32_e64 [[CMPHI:s\[[0-9]+:[0-9]+\]]], 0, v[[HI]] -; SI-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]] -; SI-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]] -; SI-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]] -; SI-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[FFBH_LO]] -; SI-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} -; SI: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} +; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}} +; GCN-DAG: v_cmp_eq_u32_e64 [[CMPHI:s\[[0-9]+:[0-9]+\]]], 0, v[[HI]] +; GCN-DAG: v_ffbh_u32_e32 [[FFBH_LO:v[0-9]+]], v[[LO]] +; GCN-DAG: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 32, [[FFBH_LO]] +; GCN-DAG: v_ffbh_u32_e32 [[FFBH_HI:v[0-9]+]], v[[HI]] +; GCN-DAG: v_cndmask_b32_e64 v[[CTLZ:[0-9]+]], [[FFBH_HI]], [[FFBH_LO]] +; GCN-DAG: v_mov_b32_e32 v[[CTLZ_HI:[0-9]+]], 0{{$}} +; GCN: {{buffer|flat}}_store_dwordx2 {{.*}}v{{\[}}[[CTLZ]]:[[CTLZ_HI]]{{\]}} define void @v_ctlz_zero_undef_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in) nounwind { %tid = call i32 @llvm.r600.read.tidig.x() %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid @@ -147,9 +146,9 @@ define void @v_ctlz_zero_undef_i64_trunc(i32 addrspace(1)* noalias %out, i64 add } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_eq_neg1: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], define void @v_ctlz_zero_undef_i32_sel_eq_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -160,9 +159,9 @@ define void @v_ctlz_zero_undef_i64_trunc(i32 addrspace(1)* noalias %out, i64 add } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_ne_neg1: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], define void @v_ctlz_zero_undef_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -173,11 +172,13 @@ define void @v_ctlz_zero_undef_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i8_sel_eq_neg1: -; SI: buffer_load_ubyte [[VAL:v[0-9]+]], -; SI: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] -; SI: buffer_store_byte [[FFBH]], - define void @v_ctlz_zero_undef_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind { - %val = load i8, i8 addrspace(1)* %valptr +; GCN: {{buffer|flat}}_load_ubyte [[VAL:v[0-9]+]], +; GCN: v_ffbh_u32_e32 [[FFBH:v[0-9]+]], [[VAL]] +; GCN: {{buffer|flat}}_store_byte [[FFBH]], +define void @v_ctlz_zero_undef_i8_sel_eq_neg1(i8 addrspace(1)* noalias %out, i8 addrspace(1)* noalias %valptr) nounwind { + %tid = call i32 @llvm.r600.read.tidig.x() + %valptr.gep = getelementptr i8, i8 addrspace(1)* %valptr, i32 %tid + %val = load i8, i8 addrspace(1)* %valptr.gep %ctlz = call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone %cmp = icmp eq i8 %val, 0 %sel = select i1 %cmp, i8 -1, i8 %ctlz @@ -186,13 +187,13 @@ define void @v_ctlz_zero_undef_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i } ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_eq_neg1_two_use: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI-DAG: v_ffbh_u32_e32 [[RESULT0:v[0-9]+]], [[VAL]] -; SI-DAG: v_cmp_eq_i32_e32 vcc, 0, [[VAL]] -; SI-DAG: v_cndmask_b32_e64 [[RESULT1:v[0-9]+]], 0, 1, vcc -; SI-DAG: buffer_store_dword [[RESULT0]] -; SI-DAG: buffer_store_byte [[RESULT1]] -; SI: s_endpgm +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN-DAG: v_ffbh_u32_e32 [[RESULT0:v[0-9]+]], [[VAL]] +; GCN-DAG: v_cmp_eq_u32_e32 vcc, 0, [[VAL]] +; GCN-DAG: v_cndmask_b32_e64 [[RESULT1:v[0-9]+]], 0, 1, vcc +; GCN-DAG: buffer_store_dword [[RESULT0]] +; GCN-DAG: buffer_store_byte [[RESULT1]] +; GCN: s_endpgm define void @v_ctlz_zero_undef_i32_sel_eq_neg1_two_use(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -205,11 +206,11 @@ define void @v_ctlz_zero_undef_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i ; Selected on wrong constant ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_eq_0: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: buffer_store_dword +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: buffer_store_dword define void @v_ctlz_zero_undef_i32_sel_eq_0(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -221,11 +222,11 @@ define void @v_ctlz_zero_undef_i32_sel_ne_neg1(i32 addrspace(1)* noalias %out, i ; Selected on wrong constant ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_ne_0: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: buffer_store_dword +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: buffer_store_dword define void @v_ctlz_zero_undef_i32_sel_ne_0(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -237,11 +238,11 @@ define void @v_ctlz_zero_undef_i32_sel_ne_0(i32 addrspace(1)* noalias %out, i32 ; Compare on wrong constant ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_eq_cmp_non0: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: buffer_store_dword +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: buffer_store_dword define void @v_ctlz_zero_undef_i32_sel_eq_cmp_non0(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone @@ -253,11 +254,11 @@ define void @v_ctlz_zero_undef_i32_sel_ne_0(i32 addrspace(1)* noalias %out, i32 ; Selected on wrong constant ; FUNC-LABEL: {{^}}v_ctlz_zero_undef_i32_sel_ne_cmp_non0: -; SI: buffer_load_dword -; SI: v_ffbh_u32_e32 -; SI: v_cmp -; SI: v_cndmask -; SI: buffer_store_dword +; GCN: buffer_load_dword +; GCN: v_ffbh_u32_e32 +; GCN: v_cmp +; GCN: v_cndmask +; GCN: buffer_store_dword define void @v_ctlz_zero_undef_i32_sel_ne_cmp_non0(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { %val = load i32, i32 addrspace(1)* %valptr %ctlz = call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone diff --git a/test/CodeGen/AMDGPU/ctpop64.ll b/test/CodeGen/AMDGPU/ctpop64.ll index d0976b7d45b8..21f366687c74 100644 --- a/test/CodeGen/AMDGPU/ctpop64.ll +++ b/test/CodeGen/AMDGPU/ctpop64.ll @@ -39,14 +39,13 @@ define void @v_ctpop_i64(i32 addrspace(1)* noalias %out, i64 addrspace(1)* noali ret void } -; FIXME: or 0 should be replaxed with copy ; FUNC-LABEL: {{^}}v_ctpop_i64_user: ; GCN: buffer_load_dwordx2 v{{\[}}[[LOVAL:[0-9]+]]:[[HIVAL:[0-9]+]]{{\]}}, ; GCN: v_bcnt_u32_b32_e64 [[MIDRESULT:v[0-9]+]], v[[LOVAL]], 0 ; SI-NEXT: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], v[[HIVAL]], [[MIDRESULT]] ; VI-NEXT: v_bcnt_u32_b32_e64 [[RESULT:v[0-9]+]], v[[HIVAL]], [[MIDRESULT]] ; GCN-DAG: v_or_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, [[RESULT]] -; GCN-DAG: v_or_b32_e64 v[[RESULT_HI:[0-9]+]], 0, s{{[0-9]+}} +; GCN-DAG: v_mov_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}} ; GCN: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} ; GCN: s_endpgm define void @v_ctpop_i64_user(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %in, i64 %s.val) nounwind { @@ -155,8 +154,8 @@ define void @s_ctpop_i128(i32 addrspace(1)* noalias %out, i128 %val) nounwind { } ; FUNC-LABEL: {{^}}s_ctpop_i65: -; GCN: s_and_b32 ; GCN: s_bcnt1_i32_b64 [[REG0:s[0-9]+]], +; GCN: s_and_b32 ; GCN: s_bcnt1_i32_b64 [[REG1:s[0-9]+]], ; GCN: s_add_i32 {{s[0-9]+}}, [[REG0]], [[REG1]] ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/cube.ll b/test/CodeGen/AMDGPU/cube.ll index ab99af5864e9..c5d1f86cea76 100644 --- a/test/CodeGen/AMDGPU/cube.ll +++ b/test/CodeGen/AMDGPU/cube.ll @@ -30,10 +30,10 @@ define void @cube(<4 x float> addrspace(1)* %out, float %a, float %b, float %c) } ; GCN-LABEL: {{^}}legacy_cube: -; GCN-DAG: v_cubeid_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} -; GCN-DAG: v_cubesc_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} -; GCN-DAG: v_cubetc_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} -; GCN-DAG: v_cubema_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN-DAG: v_cubeid_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_cubesc_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_cubetc_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_cubema_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} ; GCN: buffer_store_dwordx4 define void @legacy_cube(<4 x float> addrspace(1)* %out, <4 x float> %abcx) #1 { %cube = call <4 x float> @llvm.AMDGPU.cube(<4 x float> %abcx) diff --git a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll index dcd48c97434d..ed9b8273fa4a 100644 --- a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll +++ b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll @@ -1,12 +1,15 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s -; SI-LABEL: {{^}}load_i8_to_f32: -; SI: buffer_load_ubyte [[LOADREG:v[0-9]+]], -; SI-NOT: bfe -; SI-NOT: lshr -; SI: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[LOADREG]] -; SI: buffer_store_dword [[CONV]], +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone +declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone + +; GCN-LABEL: {{^}}load_i8_to_f32: +; GCN: buffer_load_ubyte [[LOADREG:v[0-9]+]], +; GCN-NOT: bfe +; GCN-NOT: lshr +; GCN: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[LOADREG]] +; GCN: buffer_store_dword [[CONV]], define void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind { %load = load i8, i8 addrspace(1)* %in, align 1 %cvt = uitofp i8 %load to float @@ -14,11 +17,11 @@ define void @load_i8_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* n ret void } -; SI-LABEL: {{^}}load_v2i8_to_v2f32: -; SI: buffer_load_ushort [[LD:v[0-9]+]] -; SI-DAG: v_cvt_f32_ubyte1_e32 v[[HIRESULT:[0-9]+]], [[LD]] -; SI-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[LD]] -; SI: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, +; GCN-LABEL: {{^}}load_v2i8_to_v2f32: +; GCN: buffer_load_ushort [[LD:v[0-9]+]] +; GCN-DAG: v_cvt_f32_ubyte1_e32 v[[HIRESULT:[0-9]+]], [[LD]] +; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[LD]] +; GCN: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, define void @load_v2i8_to_v2f32(<2 x float> addrspace(1)* noalias %out, <2 x i8> addrspace(1)* noalias %in) nounwind { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in, align 2 %cvt = uitofp <2 x i8> %load to <2 x float> @@ -26,13 +29,13 @@ define void @load_v2i8_to_v2f32(<2 x float> addrspace(1)* noalias %out, <2 x i8> ret void } -; SI-LABEL: {{^}}load_v3i8_to_v3f32: -; SI: buffer_load_dword [[VAL:v[0-9]+]] -; SI-NOT: v_cvt_f32_ubyte3_e32 -; SI-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, [[VAL]] -; SI-DAG: v_cvt_f32_ubyte1_e32 v[[HIRESULT:[0-9]+]], [[VAL]] -; SI-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[VAL]] -; SI: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, +; GCN-LABEL: {{^}}load_v3i8_to_v3f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN-NOT: v_cvt_f32_ubyte3_e32 +; GCN-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, [[VAL]] +; GCN-DAG: v_cvt_f32_ubyte1_e32 v[[HIRESULT:[0-9]+]], [[VAL]] +; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[VAL]] +; GCN: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, define void @load_v3i8_to_v3f32(<3 x float> addrspace(1)* noalias %out, <3 x i8> addrspace(1)* noalias %in) nounwind { %load = load <3 x i8>, <3 x i8> addrspace(1)* %in, align 4 %cvt = uitofp <3 x i8> %load to <3 x float> @@ -40,15 +43,15 @@ define void @load_v3i8_to_v3f32(<3 x float> addrspace(1)* noalias %out, <3 x i8> ret void } -; SI-LABEL: {{^}}load_v4i8_to_v4f32: -; SI: buffer_load_dword [[LOADREG:v[0-9]+]] -; SI-NOT: bfe -; SI-NOT: lshr -; SI-DAG: v_cvt_f32_ubyte3_e32 v[[HIRESULT:[0-9]+]], [[LOADREG]] -; SI-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, [[LOADREG]] -; SI-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, [[LOADREG]] -; SI-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[LOADREG]] -; SI: buffer_store_dwordx4 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, +; GCN-LABEL: {{^}}load_v4i8_to_v4f32: +; GCN: buffer_load_dword [[LOADREG:v[0-9]+]] +; GCN-NOT: bfe +; GCN-NOT: lshr +; GCN-DAG: v_cvt_f32_ubyte3_e32 v[[HIRESULT:[0-9]+]], [[LOADREG]] +; GCN-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, [[LOADREG]] +; GCN-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, [[LOADREG]] +; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[LOADREG]] +; GCN: buffer_store_dwordx4 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}}, define void @load_v4i8_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 %cvt = uitofp <4 x i8> %load to <4 x float> @@ -60,19 +63,19 @@ define void @load_v4i8_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> ; position in the word for the component. ; FIXME: Packing bytes -; SI-LABEL: {{^}}load_v4i8_to_v4f32_unaligned: -; SI: buffer_load_ubyte [[LOADREG3:v[0-9]+]] -; SI: buffer_load_ubyte [[LOADREG2:v[0-9]+]] -; SI: buffer_load_ubyte [[LOADREG1:v[0-9]+]] -; SI: buffer_load_ubyte [[LOADREG0:v[0-9]+]] -; SI-DAG: v_lshlrev_b32 -; SI-DAG: v_or_b32 -; SI-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], -; SI-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, -; SI-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, -; SI-DAG: v_cvt_f32_ubyte0_e32 v[[HIRESULT:[0-9]+]] +; GCN-LABEL: {{^}}load_v4i8_to_v4f32_unaligned: +; GCN: buffer_load_ubyte [[LOADREG3:v[0-9]+]] +; GCN: buffer_load_ubyte [[LOADREG2:v[0-9]+]] +; GCN: buffer_load_ubyte [[LOADREG1:v[0-9]+]] +; GCN: buffer_load_ubyte [[LOADREG0:v[0-9]+]] +; GCN-DAG: v_lshlrev_b32 +; GCN-DAG: v_or_b32 +; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], +; GCN-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, +; GCN-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, +; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[HIRESULT:[0-9]+]] -; SI: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 define void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 1 %cvt = uitofp <4 x i8> %load to <4 x float> @@ -80,28 +83,37 @@ define void @load_v4i8_to_v4f32_unaligned(<4 x float> addrspace(1)* noalias %out ret void } +; FIXME: Need to handle non-uniform case for function below (load without gep). ; Instructions still emitted to repack bytes for add use. -; SI-LABEL: {{^}}load_v4i8_to_v4f32_2_uses: -; SI: buffer_load_dword -; SI-DAG: v_cvt_f32_ubyte0_e32 -; SI-DAG: v_cvt_f32_ubyte1_e32 -; SI-DAG: v_cvt_f32_ubyte2_e32 -; SI-DAG: v_cvt_f32_ubyte3_e32 -; SI-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 24 -; SI-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16 +; GCN-LABEL: {{^}}load_v4i8_to_v4f32_2_uses: +; GCN: {{buffer|flat}}_load_dword +; GCN-DAG: v_cvt_f32_ubyte0_e32 +; GCN-DAG: v_cvt_f32_ubyte1_e32 +; GCN-DAG: v_cvt_f32_ubyte2_e32 +; GCN-DAG: v_cvt_f32_ubyte3_e32 + +; GCN-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 24 +; GCN-DAG: v_lshrrev_b32_e32 v{{[0-9]+}}, 16 + ; SI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, 16 ; SI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, 8 ; SI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xffff, ; SI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff00, ; SI-DAG: v_add_i32 -; SI: buffer_store_dwordx4 -; SI: buffer_store_dword +; VI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xffffff00, +; VI-DAG: v_add_u16_e32 +; VI-DAG: v_add_u16_e32 -; SI: s_endpgm +; GCN: {{buffer|flat}}_store_dwordx4 +; GCN: {{buffer|flat}}_store_dword + +; GCN: s_endpgm define void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %out2, <4 x i8> addrspace(1)* noalias %in) nounwind { - %load = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %in.ptr = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid.x + %load = load <4 x i8>, <4 x i8> addrspace(1)* %in.ptr, align 4 %cvt = uitofp <4 x i8> %load to <4 x float> store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16 %add = add <4 x i8> %load, <i8 9, i8 9, i8 9, i8 9> ; Second use of %load @@ -110,8 +122,8 @@ define void @load_v4i8_to_v4f32_2_uses(<4 x float> addrspace(1)* noalias %out, < } ; Make sure this doesn't crash. -; SI-LABEL: {{^}}load_v7i8_to_v7f32: -; SI: s_endpgm +; GCN-LABEL: {{^}}load_v7i8_to_v7f32: +; GCN: s_endpgm define void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias %out, <7 x i8> addrspace(1)* noalias %in) nounwind { %load = load <7 x i8>, <7 x i8> addrspace(1)* %in, align 1 %cvt = uitofp <7 x i8> %load to <7 x float> @@ -119,22 +131,22 @@ define void @load_v7i8_to_v7f32(<7 x float> addrspace(1)* noalias %out, <7 x i8> ret void } -; SI-LABEL: {{^}}load_v8i8_to_v8f32: -; SI: buffer_load_dwordx2 v{{\[}}[[LOLOAD:[0-9]+]]:[[HILOAD:[0-9]+]]{{\]}}, -; SI-NOT: bfe -; SI-NOT: lshr -; SI-DAG: v_cvt_f32_ubyte3_e32 v{{[0-9]+}}, v[[LOLOAD]] -; SI-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, v[[LOLOAD]] -; SI-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, v[[LOLOAD]] -; SI-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, v[[LOLOAD]] -; SI-DAG: v_cvt_f32_ubyte3_e32 v{{[0-9]+}}, v[[HILOAD]] -; SI-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, v[[HILOAD]] -; SI-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, v[[HILOAD]] -; SI-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, v[[HILOAD]] -; SI-NOT: bfe -; SI-NOT: lshr -; SI: buffer_store_dwordx4 -; SI: buffer_store_dwordx4 +; GCN-LABEL: {{^}}load_v8i8_to_v8f32: +; GCN: buffer_load_dwordx2 v{{\[}}[[LOLOAD:[0-9]+]]:[[HILOAD:[0-9]+]]{{\]}}, +; GCN-NOT: bfe +; GCN-NOT: lshr +; GCN-DAG: v_cvt_f32_ubyte3_e32 v{{[0-9]+}}, v[[LOLOAD]] +; GCN-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, v[[LOLOAD]] +; GCN-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, v[[LOLOAD]] +; GCN-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, v[[LOLOAD]] +; GCN-DAG: v_cvt_f32_ubyte3_e32 v{{[0-9]+}}, v[[HILOAD]] +; GCN-DAG: v_cvt_f32_ubyte2_e32 v{{[0-9]+}}, v[[HILOAD]] +; GCN-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, v[[HILOAD]] +; GCN-DAG: v_cvt_f32_ubyte0_e32 v{{[0-9]+}}, v[[HILOAD]] +; GCN-NOT: bfe +; GCN-NOT: lshr +; GCN: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 define void @load_v8i8_to_v8f32(<8 x float> addrspace(1)* noalias %out, <8 x i8> addrspace(1)* noalias %in) nounwind { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in, align 8 %cvt = uitofp <8 x i8> %load to <8 x float> @@ -142,11 +154,11 @@ define void @load_v8i8_to_v8f32(<8 x float> addrspace(1)* noalias %out, <8 x i8> ret void } -; SI-LABEL: {{^}}i8_zext_inreg_i32_to_f32: -; SI: buffer_load_dword [[LOADREG:v[0-9]+]], -; SI: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 2, [[LOADREG]] -; SI-NEXT: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[ADD]] -; SI: buffer_store_dword [[CONV]], +; GCN-LABEL: {{^}}i8_zext_inreg_i32_to_f32: +; GCN: buffer_load_dword [[LOADREG:v[0-9]+]], +; GCN: v_add_i32_e32 [[ADD:v[0-9]+]], vcc, 2, [[LOADREG]] +; GCN-NEXT: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[ADD]] +; GCN: buffer_store_dword [[CONV]], define void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %load = load i32, i32 addrspace(1)* %in, align 4 %add = add i32 %load, 2 @@ -156,7 +168,7 @@ define void @i8_zext_inreg_i32_to_f32(float addrspace(1)* noalias %out, i32 addr ret void } -; SI-LABEL: {{^}}i8_zext_inreg_hi1_to_f32: +; GCN-LABEL: {{^}}i8_zext_inreg_hi1_to_f32: define void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %load = load i32, i32 addrspace(1)* %in, align 4 %inreg = and i32 %load, 65280 @@ -168,7 +180,7 @@ define void @i8_zext_inreg_hi1_to_f32(float addrspace(1)* noalias %out, i32 addr ; We don't get these ones because of the zext, but instcombine removes ; them so it shouldn't really matter. -; SI-LABEL: {{^}}i8_zext_i32_to_f32: +; GCN-LABEL: {{^}}i8_zext_i32_to_f32: define void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1)* noalias %in) nounwind { %load = load i8, i8 addrspace(1)* %in, align 1 %ext = zext i8 %load to i32 @@ -177,7 +189,7 @@ define void @i8_zext_i32_to_f32(float addrspace(1)* noalias %out, i8 addrspace(1 ret void } -; SI-LABEL: {{^}}v4i8_zext_v4i32_to_v4f32: +; GCN-LABEL: {{^}}v4i8_zext_v4i32_to_v4f32: define void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %in) nounwind { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 1 %ext = zext <4 x i8> %load to <4 x i32> @@ -186,11 +198,11 @@ define void @v4i8_zext_v4i32_to_v4f32(<4 x float> addrspace(1)* noalias %out, <4 ret void } -; SI-LABEL: {{^}}extract_byte0_to_f32: -; SI: buffer_load_dword [[VAL:v[0-9]+]] -; SI-NOT: [[VAL]] -; SI: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[CONV]] +; GCN-LABEL: {{^}}extract_byte0_to_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN-NOT: [[VAL]] +; GCN: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[CONV]] define void @extract_byte0_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %val = load i32, i32 addrspace(1)* %in %and = and i32 %val, 255 @@ -199,11 +211,11 @@ define void @extract_byte0_to_f32(float addrspace(1)* noalias %out, i32 addrspac ret void } -; SI-LABEL: {{^}}extract_byte1_to_f32: -; SI: buffer_load_dword [[VAL:v[0-9]+]] -; SI-NOT: [[VAL]] -; SI: v_cvt_f32_ubyte1_e32 [[CONV:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[CONV]] +; GCN-LABEL: {{^}}extract_byte1_to_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN-NOT: [[VAL]] +; GCN: v_cvt_f32_ubyte1_e32 [[CONV:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[CONV]] define void @extract_byte1_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %val = load i32, i32 addrspace(1)* %in %srl = lshr i32 %val, 8 @@ -213,11 +225,11 @@ define void @extract_byte1_to_f32(float addrspace(1)* noalias %out, i32 addrspac ret void } -; SI-LABEL: {{^}}extract_byte2_to_f32: -; SI: buffer_load_dword [[VAL:v[0-9]+]] -; SI-NOT: [[VAL]] -; SI: v_cvt_f32_ubyte2_e32 [[CONV:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[CONV]] +; GCN-LABEL: {{^}}extract_byte2_to_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN-NOT: [[VAL]] +; GCN: v_cvt_f32_ubyte2_e32 [[CONV:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[CONV]] define void @extract_byte2_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %val = load i32, i32 addrspace(1)* %in %srl = lshr i32 %val, 16 @@ -227,11 +239,11 @@ define void @extract_byte2_to_f32(float addrspace(1)* noalias %out, i32 addrspac ret void } -; SI-LABEL: {{^}}extract_byte3_to_f32: -; SI: buffer_load_dword [[VAL:v[0-9]+]] -; SI-NOT: [[VAL]] -; SI: v_cvt_f32_ubyte3_e32 [[CONV:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[CONV]] +; GCN-LABEL: {{^}}extract_byte3_to_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN-NOT: [[VAL]] +; GCN: v_cvt_f32_ubyte3_e32 [[CONV:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[CONV]] define void @extract_byte3_to_f32(float addrspace(1)* noalias %out, i32 addrspace(1)* noalias %in) nounwind { %val = load i32, i32 addrspace(1)* %in %srl = lshr i32 %val, 24 diff --git a/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll b/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll index 2dd3a9f2a776..e7773c6e2a4f 100644 --- a/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll +++ b/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=SI -enable-no-nans-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-NONAN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -enable-no-nans-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-NONAN -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s declare float @llvm.fabs.f32(float) #1 @@ -18,7 +18,7 @@ define void @cvt_flr_i32_f32_0(i32 addrspace(1)* %out, float %x) #0 { } ; FUNC-LABEL: {{^}}cvt_flr_i32_f32_1: -; SI: v_add_f32_e64 [[TMP:v[0-9]+]], 1.0, s{{[0-9]+}} +; SI: v_add_f32_e64 [[TMP:v[0-9]+]], s{{[0-9]+}}, 1.0 ; SI-SAFE-NOT: v_cvt_flr_i32_f32 ; SI-NONAN: v_cvt_flr_i32_f32_e32 v{{[0-9]+}}, [[TMP]] ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/default-fp-mode.ll b/test/CodeGen/AMDGPU/default-fp-mode.ll index 723e3c27ad6b..28d065e3b32b 100644 --- a/test/CodeGen/AMDGPU/default-fp-mode.ll +++ b/test/CodeGen/AMDGPU/default-fp-mode.ll @@ -2,7 +2,7 @@ ; GCN-LABEL: {{^}}test_default_si: ; GCN: FloatMode: 192 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_default_si(float addrspace(1)* %out0, double addrspace(1)* %out1) #0 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -11,7 +11,7 @@ define void @test_default_si(float addrspace(1)* %out0, double addrspace(1)* %ou ; GCN-LABEL: {{^}}test_default_vi: ; GCN: FloatMode: 192 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #1 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -20,7 +20,7 @@ define void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %ou ; GCN-LABEL: {{^}}test_f64_denormals: ; GCN: FloatMode: 192 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #2 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -29,7 +29,7 @@ define void @test_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* ; GCN-LABEL: {{^}}test_f32_denormals: ; GCNL: FloatMode: 48 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_f32_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #3 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -38,7 +38,7 @@ define void @test_f32_denormals(float addrspace(1)* %out0, double addrspace(1)* ; GCN-LABEL: {{^}}test_f32_f64_denormals: ; GCN: FloatMode: 240 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_f32_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #4 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -47,13 +47,41 @@ define void @test_f32_f64_denormals(float addrspace(1)* %out0, double addrspace( ; GCN-LABEL: {{^}}test_no_denormals ; GCN: FloatMode: 0 -; GCN: IeeeMode: 0 +; GCN: IeeeMode: 1 define void @test_no_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #5 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 ret void } +; GCN-LABEL: {{^}}kill_gs_const: +; GCN: IeeeMode: 0 +define amdgpu_gs void @kill_gs_const() { +main_body: + %0 = icmp ule i32 0, 3 + %1 = select i1 %0, float 1.000000e+00, float -1.000000e+00 + call void @llvm.AMDGPU.kill(float %1) + %2 = icmp ule i32 3, 0 + %3 = select i1 %2, float 1.000000e+00, float -1.000000e+00 + call void @llvm.AMDGPU.kill(float %3) + ret void +} + +; GCN-LABEL: {{^}}kill_vcc_implicit_def: +; GCN: IeeeMode: 0 +define amdgpu_ps void @kill_vcc_implicit_def([6 x <16 x i8>] addrspace(2)* byval, [17 x <16 x i8>] addrspace(2)* byval, [17 x <4 x i32>] addrspace(2)* byval, [34 x <8 x i32>] addrspace(2)* byval, float inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, i32, float, float) { +entry: + %tmp0 = fcmp olt float %13, 0.0 + call void @llvm.AMDGPU.kill(float %14) + %tmp1 = select i1 %tmp0, float 1.0, float 0.0 + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 1, i32 1, float %tmp1, float %tmp1, float %tmp1, float %tmp1) + ret void +} + + +declare void @llvm.AMDGPU.kill(float) +declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) + attributes #0 = { nounwind "target-cpu"="tahiti" } attributes #1 = { nounwind "target-cpu"="fiji" } attributes #2 = { nounwind "target-features"="+fp64-denormals" } diff --git a/test/CodeGen/AMDGPU/detect-dead-lanes.mir b/test/CodeGen/AMDGPU/detect-dead-lanes.mir index f7f953c144da..9d70f67ef491 100644 --- a/test/CodeGen/AMDGPU/detect-dead-lanes.mir +++ b/test/CodeGen/AMDGPU/detect-dead-lanes.mir @@ -17,20 +17,19 @@ # CHECK: S_NOP 0, implicit-def %1 # CHECK: S_NOP 0, implicit-def dead %2 # CHECK: %3 = REG_SEQUENCE %0, {{[0-9]+}}, %1, {{[0-9]+}}, undef %2, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %3:sub0 -# CHECK: S_NOP 0, implicit %3:sub1 -# CHECK: S_NOP 0, implicit undef %3:sub2 -# CHECK: %4 = COPY %3:sub0_sub1 -# CHECK: %5 = COPY undef %3:sub2_sub3 -# CHECK: S_NOP 0, implicit %4:sub0 -# CHECK: S_NOP 0, implicit %4:sub1 -# CHECK: S_NOP 0, implicit undef %5:sub0 +# CHECK: S_NOP 0, implicit %3.sub0 +# CHECK: S_NOP 0, implicit %3.sub1 +# CHECK: S_NOP 0, implicit undef %3.sub2 +# CHECK: %4 = COPY %3.sub0_sub1 +# CHECK: %5 = COPY undef %3.sub2_sub3 +# CHECK: S_NOP 0, implicit %4.sub0 +# CHECK: S_NOP 0, implicit %4.sub1 +# CHECK: S_NOP 0, implicit undef %5.sub0 name: test0 -isSSA: true registers: - - { id: 0, class: sreg_32 } - - { id: 1, class: sreg_32 } - - { id: 2, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } + - { id: 1, class: sreg_32_xm0 } + - { id: 2, class: sreg_32_xm0 } - { id: 3, class: sreg_128 } - { id: 4, class: sreg_64 } - { id: 5, class: sreg_64 } @@ -40,14 +39,14 @@ body: | S_NOP 0, implicit-def %1 S_NOP 0, implicit-def %2 %3 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub3 - S_NOP 0, implicit %3:sub0 - S_NOP 0, implicit %3:sub1 - S_NOP 0, implicit %3:sub2 - %4 = COPY %3:sub0_sub1 - %5 = COPY %3:sub2_sub3 - S_NOP 0, implicit %4:sub0 - S_NOP 0, implicit %4:sub1 - S_NOP 0, implicit %5:sub0 + S_NOP 0, implicit %3.sub0 + S_NOP 0, implicit %3.sub1 + S_NOP 0, implicit %3.sub2 + %4 = COPY %3.sub0_sub1 + %5 = COPY %3.sub2_sub3 + S_NOP 0, implicit %4.sub0 + S_NOP 0, implicit %4.sub1 + S_NOP 0, implicit %5.sub0 ... --- # Check defined lanes transfer; Includes checking for some special cases like @@ -55,20 +54,20 @@ body: | # CHECK-LABEL: name: test1 # CHECK: %0 = REG_SEQUENCE %sgpr0, {{[0-9]+}}, %sgpr0, {{[0-9]+}} # CHECK: %1 = INSERT_SUBREG %0, %sgpr1, {{[0-9]+}} -# CHECK: %2 = INSERT_SUBREG %0:sub2_sub3, %sgpr42, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %1:sub0 -# CHECK: S_NOP 0, implicit undef %1:sub1 -# CHECK: S_NOP 0, implicit %1:sub2 -# CHECK: S_NOP 0, implicit %1:sub3 -# CHECK: S_NOP 0, implicit %2:sub0 -# CHECK: S_NOP 0, implicit undef %2:sub1 +# CHECK: %2 = INSERT_SUBREG %0.sub2_sub3, %sgpr42, {{[0-9]+}} +# CHECK: S_NOP 0, implicit %1.sub0 +# CHECK: S_NOP 0, implicit undef %1.sub1 +# CHECK: S_NOP 0, implicit %1.sub2 +# CHECK: S_NOP 0, implicit %1.sub3 +# CHECK: S_NOP 0, implicit %2.sub0 +# CHECK: S_NOP 0, implicit undef %2.sub1 # CHECK: %3 = IMPLICIT_DEF # CHECK: %4 = INSERT_SUBREG %0, undef %3, {{[0-9]+}} -# CHECK: S_NOP 0, implicit undef %4:sub0 -# CHECK: S_NOP 0, implicit undef %4:sub1 -# CHECK: S_NOP 0, implicit %4:sub2 -# CHECK: S_NOP 0, implicit undef %4:sub3 +# CHECK: S_NOP 0, implicit undef %4.sub0 +# CHECK: S_NOP 0, implicit undef %4.sub1 +# CHECK: S_NOP 0, implicit %4.sub2 +# CHECK: S_NOP 0, implicit undef %4.sub3 # CHECK: %5 = EXTRACT_SUBREG %0, {{[0-9]+}} # CHECK: %6 = EXTRACT_SUBREG %5, {{[0-9]+}} @@ -84,37 +83,36 @@ body: | # CHECK: %10 = EXTRACT_SUBREG undef %0, {{[0-9]+}} # CHECK: S_NOP 0, implicit undef %10 name: test1 -isSSA: true registers: - { id: 0, class: sreg_128 } - { id: 1, class: sreg_128 } - { id: 2, class: sreg_64 } - - { id: 3, class: sreg_32 } + - { id: 3, class: sreg_32_xm0 } - { id: 4, class: sreg_128 } - { id: 5, class: sreg_64 } - - { id: 6, class: sreg_32 } - - { id: 7, class: sreg_32 } + - { id: 6, class: sreg_32_xm0 } + - { id: 7, class: sreg_32_xm0 } - { id: 8, class: sreg_64 } - - { id: 9, class: sreg_32 } + - { id: 9, class: sreg_32_xm0 } - { id: 10, class: sreg_128 } body: | bb.0: %0 = REG_SEQUENCE %sgpr0, %subreg.sub0, %sgpr0, %subreg.sub2 %1 = INSERT_SUBREG %0, %sgpr1, %subreg.sub3 - %2 = INSERT_SUBREG %0:sub2_sub3, %sgpr42, %subreg.sub0 - S_NOP 0, implicit %1:sub0 - S_NOP 0, implicit %1:sub1 - S_NOP 0, implicit %1:sub2 - S_NOP 0, implicit %1:sub3 - S_NOP 0, implicit %2:sub0 - S_NOP 0, implicit %2:sub1 + %2 = INSERT_SUBREG %0.sub2_sub3, %sgpr42, %subreg.sub0 + S_NOP 0, implicit %1.sub0 + S_NOP 0, implicit %1.sub1 + S_NOP 0, implicit %1.sub2 + S_NOP 0, implicit %1.sub3 + S_NOP 0, implicit %2.sub0 + S_NOP 0, implicit %2.sub1 %3 = IMPLICIT_DEF %4 = INSERT_SUBREG %0, %3, %subreg.sub0 - S_NOP 0, implicit %4:sub0 - S_NOP 0, implicit %4:sub1 - S_NOP 0, implicit %4:sub2 - S_NOP 0, implicit %4:sub3 + S_NOP 0, implicit %4.sub0 + S_NOP 0, implicit %4.sub1 + S_NOP 0, implicit %4.sub2 + S_NOP 0, implicit %4.sub3 %5 = EXTRACT_SUBREG %0, %subreg.sub0_sub1 %6 = EXTRACT_SUBREG %5, %subreg.sub0 @@ -138,8 +136,8 @@ body: | # CHECK: S_NOP 0, implicit-def %1 # CHECK: S_NOP 0, implicit-def %2 # CHECK: %3 = REG_SEQUENCE undef %0, {{[0-9]+}}, %1, {{[0-9]+}}, %2, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %3:sub1 -# CHECK: S_NOP 0, implicit %3:sub3 +# CHECK: S_NOP 0, implicit %3.sub1 +# CHECK: S_NOP 0, implicit %3.sub3 # CHECK: S_NOP 0, implicit-def %4 # CHECK: S_NOP 0, implicit-def dead %5 @@ -149,28 +147,27 @@ body: | # CHECK: S_NOP 0, implicit-def dead %7 # CHECK: S_NOP 0, implicit-def %8 # CHECK: %9 = INSERT_SUBREG undef %7, %8, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %9:sub2 +# CHECK: S_NOP 0, implicit %9.sub2 # CHECK: S_NOP 0, implicit-def %10 # CHECK: S_NOP 0, implicit-def dead %11 # CHECK: %12 = INSERT_SUBREG %10, undef %11, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %12:sub3 +# CHECK: S_NOP 0, implicit %12.sub3 # CHECK: S_NOP 0, implicit-def %13 # CHECK: S_NOP 0, implicit-def dead %14 # CHECK: %15 = REG_SEQUENCE %13, {{[0-9]+}}, undef %14, {{[0-9]+}} # CHECK: %16 = EXTRACT_SUBREG %15, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %16:sub1 +# CHECK: S_NOP 0, implicit %16.sub1 name: test2 -isSSA: true registers: - - { id: 0, class: sreg_32 } - - { id: 1, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } + - { id: 1, class: sreg_32_xm0 } - { id: 2, class: sreg_64 } - { id: 3, class: sreg_128 } - - { id: 4, class: sreg_32 } - - { id: 5, class: sreg_32 } + - { id: 4, class: sreg_32_xm0 } + - { id: 5, class: sreg_32_xm0 } - { id: 6, class: sreg_64 } - { id: 7, class: sreg_128 } - { id: 8, class: sreg_64 } @@ -188,8 +185,8 @@ body: | S_NOP 0, implicit-def %1 S_NOP 0, implicit-def %2 %3 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2_sub3 - S_NOP 0, implicit %3:sub1 - S_NOP 0, implicit %3:sub3 + S_NOP 0, implicit %3.sub1 + S_NOP 0, implicit %3.sub3 S_NOP 0, implicit-def %4 S_NOP 0, implicit-def %5 @@ -199,18 +196,18 @@ body: | S_NOP 0, implicit-def %7 S_NOP 0, implicit-def %8 %9 = INSERT_SUBREG %7, %8, %subreg.sub2_sub3 - S_NOP 0, implicit %9:sub2 + S_NOP 0, implicit %9.sub2 S_NOP 0, implicit-def %10 S_NOP 0, implicit-def %11 %12 = INSERT_SUBREG %10, %11, %subreg.sub0_sub1 - S_NOP 0, implicit %12:sub3 + S_NOP 0, implicit %12.sub3 S_NOP 0, implicit-def %13 S_NOP 0, implicit-def %14 %15 = REG_SEQUENCE %13, %subreg.sub0_sub1, %14, %subreg.sub2_sub3 %16 = EXTRACT_SUBREG %15, %subreg.sub0_sub1 - S_NOP 0, implicit %16:sub1 + S_NOP 0, implicit %16.sub1 ... --- # Check that copies to physregs use all lanes, copies from physregs define all @@ -221,7 +218,6 @@ body: | # CHECK: %1 = COPY %vcc # CHECK: S_NOP 0, implicit %1 name: test3 -isSSA: true tracksRegLiveness: true registers: - { id: 0, class: sreg_64 } @@ -242,7 +238,6 @@ body: | # CHECK: %1 = IMPLICIT_DEF # CHECK: S_NOP 0, implicit undef %1 name: test4 -isSSA: true tracksRegLiveness: true registers: - { id: 0, class: sreg_64 } @@ -261,18 +256,17 @@ body: | # CHECK-LABEL: name: test5 # CHECK: S_NOP 0, implicit-def %0 # CHECK: %1 = REG_SEQUENCE undef %0, {{[0-9]+}}, %0, {{[0-9]+}} -# CHECK: S_NOP 0, implicit %1:sub1 +# CHECK: S_NOP 0, implicit %1.sub1 name: test5 -isSSA: true tracksRegLiveness: true registers: - - { id: 0, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } - { id: 1, class: sreg_64 } body: | bb.0: S_NOP 0, implicit-def %0 %1 = REG_SEQUENCE %0, %subreg.sub0, %0, %subreg.sub1 - S_NOP 0, implicit %1:sub1 + S_NOP 0, implicit %1.sub1 ... --- # Check "optimistic" dataflow fixpoint in phi-loops. @@ -287,15 +281,14 @@ body: | # CHECK: %4 = PHI %3, %bb.0, %5, %bb.1 # CHECK: bb.2: -# CHECK: S_NOP 0, implicit %4:sub0 -# CHECK: S_NOP 0, implicit undef %4:sub3 +# CHECK: S_NOP 0, implicit %4.sub0 +# CHECK: S_NOP 0, implicit undef %4.sub3 name: loop0 -isSSA: true tracksRegLiveness: true registers: - - { id: 0, class: sreg_32 } - - { id: 1, class: sreg_32 } - - { id: 2, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } + - { id: 1, class: sreg_32_xm0 } + - { id: 2, class: sreg_32_xm0 } - { id: 3, class: sreg_128 } - { id: 4, class: sreg_128 } - { id: 5, class: sreg_128 } @@ -313,14 +306,14 @@ body: | %4 = PHI %3, %bb.0, %5, %bb.1 ; let's swiffle some lanes around for fun... - %5 = REG_SEQUENCE %4:sub0, %subreg.sub0, %4:sub2, %subreg.sub1, %4:sub1, %subreg.sub2, %4:sub3, %subreg.sub3 + %5 = REG_SEQUENCE %4.sub0, %subreg.sub0, %4.sub2, %subreg.sub1, %4.sub1, %subreg.sub2, %4.sub3, %subreg.sub3 S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc S_BRANCH %bb.2 bb.2: - S_NOP 0, implicit %4:sub0 - S_NOP 0, implicit %4:sub3 + S_NOP 0, implicit %4.sub0 + S_NOP 0, implicit %4.sub3 ... --- # Check a loop that needs to be traversed multiple times to reach the fixpoint @@ -339,18 +332,17 @@ body: | # CHECK: bb.1: # CHECK: %5 = PHI %4, %bb.0, %6, %bb.1 -# CHECK: %6 = REG_SEQUENCE %5:sub1, {{[0-9]+}}, %5:sub3, {{[0-9]+}}, undef %5:sub2, {{[0-9]+}}, %5:sub0, {{[0-9]+}} +# CHECK: %6 = REG_SEQUENCE %5.sub1, {{[0-9]+}}, %5.sub3, {{[0-9]+}}, undef %5.sub2, {{[0-9]+}}, %5.sub0, {{[0-9]+}} # CHECK: bb.2: -# CHECK: S_NOP 0, implicit %6:sub3 +# CHECK: S_NOP 0, implicit %6.sub3 name: loop1 -isSSA: true tracksRegLiveness: true registers: - - { id: 0, class: sreg_32 } - - { id: 1, class: sreg_32 } - - { id: 2, class: sreg_32 } - - { id: 3, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } + - { id: 1, class: sreg_32_xm0 } + - { id: 2, class: sreg_32_xm0 } + - { id: 3, class: sreg_32_xm0 } - { id: 4, class: sreg_128 } - { id: 5, class: sreg_128 } - { id: 6, class: sreg_128 } @@ -369,13 +361,13 @@ body: | %5 = PHI %4, %bb.0, %6, %bb.1 ; rotate lanes, but skip sub2 lane... - %6 = REG_SEQUENCE %5:sub1, %subreg.sub0, %5:sub3, %subreg.sub1, %5:sub2, %subreg.sub2, %5:sub0, %subreg.sub3 + %6 = REG_SEQUENCE %5.sub1, %subreg.sub0, %5.sub3, %subreg.sub1, %5.sub2, %subreg.sub2, %5.sub0, %subreg.sub3 S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc S_BRANCH %bb.2 bb.2: - S_NOP 0, implicit %6:sub3 + S_NOP 0, implicit %6.sub3 ... --- # Similar to loop1 test, but check for fixpoint of defined lanes. @@ -388,18 +380,17 @@ body: | # CHECK: bb.1: # CHECK: %2 = PHI %1, %bb.0, %3, %bb.1 -# CHECK: %3 = REG_SEQUENCE %2:sub3, {{[0-9]+}}, undef %2:sub1, {{[0-9]+}}, %2:sub0, {{[0-9]+}}, %2:sub2, {{[0-9]+}} +# CHECK: %3 = REG_SEQUENCE %2.sub3, {{[0-9]+}}, undef %2.sub1, {{[0-9]+}}, %2.sub0, {{[0-9]+}}, %2.sub2, {{[0-9]+}} # CHECK: bb.2: -# CHECK: S_NOP 0, implicit %2:sub0 -# CHECK: S_NOP 0, implicit undef %2:sub1 -# CHECK: S_NOP 0, implicit %2:sub2 -# CHECK: S_NOP 0, implicit %2:sub3 +# CHECK: S_NOP 0, implicit %2.sub0 +# CHECK: S_NOP 0, implicit undef %2.sub1 +# CHECK: S_NOP 0, implicit %2.sub2 +# CHECK: S_NOP 0, implicit %2.sub3 name: loop2 -isSSA: true tracksRegLiveness: true registers: - - { id: 0, class: sreg_32 } + - { id: 0, class: sreg_32_xm0 } - { id: 1, class: sreg_128 } - { id: 2, class: sreg_128 } - { id: 3, class: sreg_128 } @@ -415,14 +406,14 @@ body: | %2 = PHI %1, %bb.0, %3, %bb.1 ; rotate subreg lanes, skipping sub1 - %3 = REG_SEQUENCE %2:sub3, %subreg.sub0, %2:sub1, %subreg.sub1, %2:sub0, %subreg.sub2, %2:sub2, %subreg.sub3 + %3 = REG_SEQUENCE %2.sub3, %subreg.sub0, %2.sub1, %subreg.sub1, %2.sub0, %subreg.sub2, %2.sub2, %subreg.sub3 S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc S_BRANCH %bb.2 bb.2: - S_NOP 0, implicit %2:sub0 - S_NOP 0, implicit undef %2:sub1 - S_NOP 0, implicit %2:sub2 - S_NOP 0, implicit %2:sub3 + S_NOP 0, implicit %2.sub0 + S_NOP 0, implicit undef %2.sub1 + S_NOP 0, implicit %2.sub2 + S_NOP 0, implicit %2.sub3 ... diff --git a/test/CodeGen/AMDGPU/ds_read2.ll b/test/CodeGen/AMDGPU/ds_read2.ll index 6e30cff9609d..9a313230e303 100644 --- a/test/CodeGen/AMDGPU/ds_read2.ll +++ b/test/CodeGen/AMDGPU/ds_read2.ll @@ -493,6 +493,46 @@ define void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) ret void } +; SI-LABEL: ds_read_diff_base_interleaving +; SI-NOT: ds_read_b32 +define amdgpu_kernel void @ds_read_diff_base_interleaving( + float addrspace(1)* nocapture %arg, + [4 x [4 x float]] addrspace(3)* %arg1, + [4 x [4 x float]] addrspace(3)* %arg2, + [4 x [4 x float]] addrspace(3)* %arg3, + [4 x [4 x float]] addrspace(3)* %arg4) #1 { +bb: + %tmp = getelementptr float, float addrspace(1)* %arg, i64 10 + %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.x() #2 + %tmp6 = tail call i32 @llvm.amdgcn.workitem.id.y() #2 + %tmp7 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 0 + %tmp8 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 0, i32 %tmp5 + %tmp9 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 0 + %tmp10 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 0, i32 %tmp5 + %tmp11 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 1 + %tmp12 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 1, i32 %tmp5 + %tmp13 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 1 + %tmp14 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 1, i32 %tmp5 + %tmp15 = load float, float addrspace(3)* %tmp7 + %tmp16 = load float, float addrspace(3)* %tmp8 + %tmp17 = fmul float %tmp15, %tmp16 + %tmp18 = fadd float 2.000000e+00, %tmp17 + %tmp19 = load float, float addrspace(3)* %tmp9 + %tmp20 = load float, float addrspace(3)* %tmp10 + %tmp21 = fmul float %tmp19, %tmp20 + %tmp22 = fsub float %tmp18, %tmp21 + %tmp23 = load float, float addrspace(3)* %tmp11 + %tmp24 = load float, float addrspace(3)* %tmp12 + %tmp25 = fmul float %tmp23, %tmp24 + %tmp26 = fsub float %tmp22, %tmp25 + %tmp27 = load float, float addrspace(3)* %tmp13 + %tmp28 = load float, float addrspace(3)* %tmp14 + %tmp29 = fmul float %tmp27, %tmp28 + %tmp30 = fsub float %tmp26, %tmp29 + store float %tmp30, float addrspace(1)* %tmp + ret void +} + ; Function Attrs: nounwind readnone declare i32 @llvm.amdgcn.workgroup.id.x() #1 diff --git a/test/CodeGen/AMDGPU/ds_read2_offset_order.ll b/test/CodeGen/AMDGPU/ds_read2_offset_order.ll index 57e190e0cca0..4a3f3fb99700 100644 --- a/test/CodeGen/AMDGPU/ds_read2_offset_order.ll +++ b/test/CodeGen/AMDGPU/ds_read2_offset_order.ll @@ -1,17 +1,15 @@ -; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -strict-whitespace -check-prefix=SI %s - +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=SI %s @lds = addrspace(3) global [512 x float] undef, align 4 ; offset0 is larger than offset1 ; SI-LABEL: {{^}}offset_order: - -; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:2 offset1:3 -; SI: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:14 offset1:12 -; SI: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:44 - +; SI-DAG: ds_read2st64_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset1:4{{$}} +; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:2 offset1:3 +; SI-DAG: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:56 +; SI-DAG: ds_read2_b32 v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} offset0:11 offset1:12 define void @offset_order(float addrspace(1)* %out) { entry: %ptr0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 0 diff --git a/test/CodeGen/AMDGPU/ds_read2st64.ll b/test/CodeGen/AMDGPU/ds_read2st64.ll index 7a8a206033ba..99f01b4f2622 100644 --- a/test/CodeGen/AMDGPU/ds_read2st64.ll +++ b/test/CodeGen/AMDGPU/ds_read2st64.ll @@ -44,9 +44,9 @@ define void @simple_read2st64_f32_1_2(float addrspace(1)* %out, float addrspace( } ; SI-LABEL: @simple_read2st64_f32_max_offset -; SI: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:255 offset1:1 +; SI: ds_read2st64_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1 offset1:255 ; SI: s_waitcnt lgkmcnt(0) -; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] +; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[HI_VREG]], v[[LO_VREG]] ; SI: buffer_store_dword [[RESULT]] ; SI: s_endpgm define void @simple_read2st64_f32_max_offset(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { @@ -176,9 +176,9 @@ define void @misaligned_read2st64_f64(double addrspace(1)* %out, double addrspac ; The maximum is not the usual 0xff because 0xff * 8 * 64 > 0xffff ; SI-LABEL: @simple_read2st64_f64_max_offset -; SI: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:127 offset1:4 +; SI: ds_read2st64_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:4 offset1:127 ; SI: s_waitcnt lgkmcnt(0) -; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+}}:[[HI_VREG]]{{\]}}, v{{\[}}[[LO_VREG]]:{{[0-9]+\]}} +; SI: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}} ; SI: buffer_store_dwordx2 [[RESULT]] ; SI: s_endpgm define void @simple_read2st64_f64_max_offset(double addrspace(1)* %out, double addrspace(3)* %lds) #0 { diff --git a/test/CodeGen/AMDGPU/ds_write2.ll b/test/CodeGen/AMDGPU/ds_write2.ll index 45fcc01b2add..ae230dac9378 100644 --- a/test/CodeGen/AMDGPU/ds_write2.ll +++ b/test/CodeGen/AMDGPU/ds_write2.ll @@ -179,8 +179,8 @@ define void @simple_write2_two_val_too_far_f32(float addrspace(1)* %C, float add } ; SI-LABEL: @simple_write2_two_val_f32_x2 -; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL0]] offset1:11 -; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL1:v[0-9]+]], [[VAL1]] offset0:8 offset1:27 +; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset1:8 +; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 ; SI: s_endpgm define void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -209,8 +209,8 @@ define void @simple_write2_two_val_f32_x2(float addrspace(1)* %C, float addrspac } ; SI-LABEL: @simple_write2_two_val_f32_x2_nonzero_base -; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL0]] offset0:3 offset1:11 -; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL1:v[0-9]+]], [[VAL1]] offset0:8 offset1:27 +; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0:v[0-9]+]], [[VAL1:v[0-9]+]] offset0:3 offset1:8 +; SI: ds_write2_b32 [[BASEADDR:v[0-9]+]], [[VAL0]], [[VAL1]] offset0:11 offset1:27 ; SI: s_endpgm define void @simple_write2_two_val_f32_x2_nonzero_base(float addrspace(1)* %C, float addrspace(1)* %in0, float addrspace(1)* %in1) #0 { %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 diff --git a/test/CodeGen/AMDGPU/elf.ll b/test/CodeGen/AMDGPU/elf.ll index c62e57c6eaac..e527b8511fda 100644 --- a/test/CodeGen/AMDGPU/elf.ll +++ b/test/CodeGen/AMDGPU/elf.ll @@ -21,7 +21,7 @@ ; CONFIG: .section .AMDGPU.config ; CONFIG-NEXT: .long 45096 ; TYPICAL-NEXT: .long 0 -; TONGA-NEXT: .long 576 +; TONGA-NEXT: .long 704 ; CONFIG: .p2align 8 ; CONFIG: test: define amdgpu_ps void @test(i32 %p) { diff --git a/test/CodeGen/AMDGPU/else.ll b/test/CodeGen/AMDGPU/else.ll new file mode 100644 index 000000000000..ef1e64763d4a --- /dev/null +++ b/test/CodeGen/AMDGPU/else.ll @@ -0,0 +1,60 @@ +; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s + +; CHECK-LABEL: {{^}}else_no_execfix: +; CHECK: ; %Flow +; CHECK-NEXT: s_or_saveexec_b64 [[DST:s\[[0-9]+:[0-9]+\]]], +; CHECK-NEXT: s_xor_b64 exec, exec, [[DST]] +; CHECK-NEXT: ; mask branch +define amdgpu_ps float @else_no_execfix(i32 %z, float %v) { +main_body: + %cc = icmp sgt i32 %z, 5 + br i1 %cc, label %if, label %else + +if: + %v.if = fmul float %v, 2.0 + br label %end + +else: + %v.else = fmul float %v, 3.0 + br label %end + +end: + %r = phi float [ %v.if, %if ], [ %v.else, %else ] + ret float %r +} + +; CHECK-LABEL: {{^}}else_execfix_leave_wqm: +; CHECK: ; BB#0: +; CHECK-NEXT: s_mov_b64 [[INIT_EXEC:s\[[0-9]+:[0-9]+\]]], exec +; CHECK: ; %Flow +; CHECK-NEXT: s_or_saveexec_b64 [[DST:s\[[0-9]+:[0-9]+\]]], +; CHECK-NEXT: s_and_b64 exec, exec, [[INIT_EXEC]] +; CHECK-NEXT: s_and_b64 [[AND_INIT:s\[[0-9]+:[0-9]+\]]], exec, [[DST]] +; CHECK-NEXT: s_xor_b64 exec, exec, [[AND_INIT]] +; CHECK-NEXT: ; mask branch +define amdgpu_ps void @else_execfix_leave_wqm(i32 %z, float %v) { +main_body: + %cc = icmp sgt i32 %z, 5 + br i1 %cc, label %if, label %else + +if: + %v.if = fmul float %v, 2.0 + br label %end + +else: + %c = fmul float %v, 3.0 + %c.i = bitcast float %c to i32 + %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c.i, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %v.else = extractelement <4 x float> %tex, i32 0 + br label %end + +end: + %r = phi float [ %v.if, %if ], [ %v.else, %else ] + call void @llvm.amdgcn.buffer.store.f32(float %r, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0) + ret void +} + +declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) nounwind + +declare <4 x float> @llvm.SI.image.sample.i32(i32, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) nounwind readnone diff --git a/test/CodeGen/AMDGPU/exceed-max-sgprs.ll b/test/CodeGen/AMDGPU/exceed-max-sgprs.ll new file mode 100644 index 000000000000..8ef54b9e95d3 --- /dev/null +++ b/test/CodeGen/AMDGPU/exceed-max-sgprs.ll @@ -0,0 +1,102 @@ +; RUN: not llc -march=amdgcn -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR %s + +; ERROR: error: scalar registers limit of 104 exceeded (106) in use_too_many_sgprs_tahiti +define void @use_too_many_sgprs_tahiti() #0 { + call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () + call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () + call void asm sideeffect "", "~{SGPR16_SGPR17_SGPR18_SGPR19_SGPR20_SGPR21_SGPR22_SGPR23}" () + call void asm sideeffect "", "~{SGPR24_SGPR25_SGPR26_SGPR27_SGPR28_SGPR29_SGPR30_SGPR31}" () + call void asm sideeffect "", "~{SGPR32_SGPR33_SGPR34_SGPR35_SGPR36_SGPR37_SGPR38_SGPR39}" () + call void asm sideeffect "", "~{SGPR40_SGPR41_SGPR42_SGPR43_SGPR44_SGPR45_SGPR46_SGPR47}" () + call void asm sideeffect "", "~{SGPR48_SGPR49_SGPR50_SGPR51_SGPR52_SGPR53_SGPR54_SGPR55}" () + call void asm sideeffect "", "~{SGPR56_SGPR57_SGPR58_SGPR59_SGPR60_SGPR61_SGPR62_SGPR63}" () + call void asm sideeffect "", "~{SGPR64_SGPR65_SGPR66_SGPR67_SGPR68_SGPR69_SGPR70_SGPR71}" () + call void asm sideeffect "", "~{SGPR72_SGPR73_SGPR74_SGPR75_SGPR76_SGPR77_SGPR78_SGPR79}" () + call void asm sideeffect "", "~{SGPR80_SGPR81_SGPR82_SGPR83_SGPR84_SGPR85_SGPR86_SGPR87}" () + call void asm sideeffect "", "~{SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95}" () + call void asm sideeffect "", "~{SGPR96_SGPR97_SGPR98_SGPR99_SGPR100_SGPR101_SGPR102_SGPR103}" () + call void asm sideeffect "", "~{VCC}" () + ret void +} + +; ERROR: error: scalar registers limit of 104 exceeded (106) in use_too_many_sgprs_bonaire +define void @use_too_many_sgprs_bonaire() #1 { + call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () + call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () + call void asm sideeffect "", "~{SGPR16_SGPR17_SGPR18_SGPR19_SGPR20_SGPR21_SGPR22_SGPR23}" () + call void asm sideeffect "", "~{SGPR24_SGPR25_SGPR26_SGPR27_SGPR28_SGPR29_SGPR30_SGPR31}" () + call void asm sideeffect "", "~{SGPR32_SGPR33_SGPR34_SGPR35_SGPR36_SGPR37_SGPR38_SGPR39}" () + call void asm sideeffect "", "~{SGPR40_SGPR41_SGPR42_SGPR43_SGPR44_SGPR45_SGPR46_SGPR47}" () + call void asm sideeffect "", "~{SGPR48_SGPR49_SGPR50_SGPR51_SGPR52_SGPR53_SGPR54_SGPR55}" () + call void asm sideeffect "", "~{SGPR56_SGPR57_SGPR58_SGPR59_SGPR60_SGPR61_SGPR62_SGPR63}" () + call void asm sideeffect "", "~{SGPR64_SGPR65_SGPR66_SGPR67_SGPR68_SGPR69_SGPR70_SGPR71}" () + call void asm sideeffect "", "~{SGPR72_SGPR73_SGPR74_SGPR75_SGPR76_SGPR77_SGPR78_SGPR79}" () + call void asm sideeffect "", "~{SGPR80_SGPR81_SGPR82_SGPR83_SGPR84_SGPR85_SGPR86_SGPR87}" () + call void asm sideeffect "", "~{SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95}" () + call void asm sideeffect "", "~{SGPR96_SGPR97_SGPR98_SGPR99_SGPR100_SGPR101_SGPR102_SGPR103}" () + call void asm sideeffect "", "~{VCC}" () + ret void +} + +; ERROR: error: scalar registers limit of 104 exceeded (106) in use_too_many_sgprs_bonaire_flat_scr +define void @use_too_many_sgprs_bonaire_flat_scr() #1 { + call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () + call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () + call void asm sideeffect "", "~{SGPR16_SGPR17_SGPR18_SGPR19_SGPR20_SGPR21_SGPR22_SGPR23}" () + call void asm sideeffect "", "~{SGPR24_SGPR25_SGPR26_SGPR27_SGPR28_SGPR29_SGPR30_SGPR31}" () + call void asm sideeffect "", "~{SGPR32_SGPR33_SGPR34_SGPR35_SGPR36_SGPR37_SGPR38_SGPR39}" () + call void asm sideeffect "", "~{SGPR40_SGPR41_SGPR42_SGPR43_SGPR44_SGPR45_SGPR46_SGPR47}" () + call void asm sideeffect "", "~{SGPR48_SGPR49_SGPR50_SGPR51_SGPR52_SGPR53_SGPR54_SGPR55}" () + call void asm sideeffect "", "~{SGPR56_SGPR57_SGPR58_SGPR59_SGPR60_SGPR61_SGPR62_SGPR63}" () + call void asm sideeffect "", "~{SGPR64_SGPR65_SGPR66_SGPR67_SGPR68_SGPR69_SGPR70_SGPR71}" () + call void asm sideeffect "", "~{SGPR72_SGPR73_SGPR74_SGPR75_SGPR76_SGPR77_SGPR78_SGPR79}" () + call void asm sideeffect "", "~{SGPR80_SGPR81_SGPR82_SGPR83_SGPR84_SGPR85_SGPR86_SGPR87}" () + call void asm sideeffect "", "~{SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95}" () + call void asm sideeffect "", "~{SGPR96_SGPR97_SGPR98_SGPR99_SGPR100_SGPR101_SGPR102_SGPR103}" () + call void asm sideeffect "", "~{VCC}" () + call void asm sideeffect "", "~{FLAT_SCR}" () + ret void +} + +; ERROR: error: scalar registers limit of 96 exceeded (98) in use_too_many_sgprs_iceland +define void @use_too_many_sgprs_iceland() #2 { + call void asm sideeffect "", "~{VCC}" () + call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () + call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () + call void asm sideeffect "", "~{SGPR16_SGPR17_SGPR18_SGPR19_SGPR20_SGPR21_SGPR22_SGPR23}" () + call void asm sideeffect "", "~{SGPR24_SGPR25_SGPR26_SGPR27_SGPR28_SGPR29_SGPR30_SGPR31}" () + call void asm sideeffect "", "~{SGPR32_SGPR33_SGPR34_SGPR35_SGPR36_SGPR37_SGPR38_SGPR39}" () + call void asm sideeffect "", "~{SGPR40_SGPR41_SGPR42_SGPR43_SGPR44_SGPR45_SGPR46_SGPR47}" () + call void asm sideeffect "", "~{SGPR48_SGPR49_SGPR50_SGPR51_SGPR52_SGPR53_SGPR54_SGPR55}" () + call void asm sideeffect "", "~{SGPR56_SGPR57_SGPR58_SGPR59_SGPR60_SGPR61_SGPR62_SGPR63}" () + call void asm sideeffect "", "~{SGPR64_SGPR65_SGPR66_SGPR67_SGPR68_SGPR69_SGPR70_SGPR71}" () + call void asm sideeffect "", "~{SGPR72_SGPR73_SGPR74_SGPR75_SGPR76_SGPR77_SGPR78_SGPR79}" () + call void asm sideeffect "", "~{SGPR80_SGPR81_SGPR82_SGPR83_SGPR84_SGPR85_SGPR86_SGPR87}" () + call void asm sideeffect "", "~{SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95}" () + ret void +} + +; ERROR: error: addressable scalar registers limit of 102 exceeded (103) in use_too_many_sgprs_fiji +define void @use_too_many_sgprs_fiji() #3 { + call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () + call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () + call void asm sideeffect "", "~{SGPR16_SGPR17_SGPR18_SGPR19_SGPR20_SGPR21_SGPR22_SGPR23}" () + call void asm sideeffect "", "~{SGPR24_SGPR25_SGPR26_SGPR27_SGPR28_SGPR29_SGPR30_SGPR31}" () + call void asm sideeffect "", "~{SGPR32_SGPR33_SGPR34_SGPR35_SGPR36_SGPR37_SGPR38_SGPR39}" () + call void asm sideeffect "", "~{SGPR40_SGPR41_SGPR42_SGPR43_SGPR44_SGPR45_SGPR46_SGPR47}" () + call void asm sideeffect "", "~{SGPR48_SGPR49_SGPR50_SGPR51_SGPR52_SGPR53_SGPR54_SGPR55}" () + call void asm sideeffect "", "~{SGPR56_SGPR57_SGPR58_SGPR59_SGPR60_SGPR61_SGPR62_SGPR63}" () + call void asm sideeffect "", "~{SGPR64_SGPR65_SGPR66_SGPR67_SGPR68_SGPR69_SGPR70_SGPR71}" () + call void asm sideeffect "", "~{SGPR72_SGPR73_SGPR74_SGPR75_SGPR76_SGPR77_SGPR78_SGPR79}" () + call void asm sideeffect "", "~{SGPR80_SGPR81_SGPR82_SGPR83_SGPR84_SGPR85_SGPR86_SGPR87}" () + call void asm sideeffect "", "~{SGPR88_SGPR89_SGPR90_SGPR91_SGPR92_SGPR93_SGPR94_SGPR95}" () + call void asm sideeffect "", "~{SGPR96_SGPR97_SGPR98_SGPR99}" () + call void asm sideeffect "", "~{SGPR100_SGPR101}" () + call void asm sideeffect "", "~{SGPR102}" () + ret void +} + +attributes #0 = { nounwind "target-cpu"="tahiti" } +attributes #1 = { nounwind "target-cpu"="bonaire" } +attributes #2 = { nounwind "target-cpu"="iceland" } +attributes #3 = { nounwind "target-cpu"="fiji" } diff --git a/test/CodeGen/AMDGPU/extend-bit-ops-i16.ll b/test/CodeGen/AMDGPU/extend-bit-ops-i16.ll new file mode 100644 index 000000000000..cf384da2c5be --- /dev/null +++ b/test/CodeGen/AMDGPU/extend-bit-ops-i16.ll @@ -0,0 +1,50 @@ +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=GCN + +; GCN-LABEL: and_zext: +; GCN: v_and_b32_e32 [[VAL16:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[VAL16]] +define void @and_zext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) { + %id = call i32 @llvm.amdgcn.workitem.id.x() #1 + %ptr = getelementptr i16, i16 addrspace(1)* %in, i32 %id + %a = load i16, i16 addrspace(1)* %in + %b = load i16, i16 addrspace(1)* %ptr + %c = add i16 %a, %b + %val16 = and i16 %c, %a + %val32 = zext i16 %val16 to i32 + store i32 %val32, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: or_zext: +; GCN: v_or_b32_e32 [[VAL16:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[VAL16]] +define void @or_zext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) { + %id = call i32 @llvm.amdgcn.workitem.id.x() #1 + %ptr = getelementptr i16, i16 addrspace(1)* %in, i32 %id + %a = load i16, i16 addrspace(1)* %in + %b = load i16, i16 addrspace(1)* %ptr + %c = add i16 %a, %b + %val16 = or i16 %c, %a + %val32 = zext i16 %val16 to i32 + store i32 %val32, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: xor_zext: +; GCN: v_xor_b32_e32 [[VAL16:v[0-9]+]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[VAL16]] +define void @xor_zext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) { + %id = call i32 @llvm.amdgcn.workitem.id.x() #1 + %ptr = getelementptr i16, i16 addrspace(1)* %in, i32 %id + %a = load i16, i16 addrspace(1)* %in + %b = load i16, i16 addrspace(1)* %ptr + %c = add i16 %a, %b + %val16 = xor i16 %c, %a + %val32 = zext i16 %val16 to i32 + store i32 %val32, i32 addrspace(1)* %out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/extload-align.ll b/test/CodeGen/AMDGPU/extload-align.ll new file mode 100644 index 000000000000..9d2eb74c7ba9 --- /dev/null +++ b/test/CodeGen/AMDGPU/extload-align.ll @@ -0,0 +1,23 @@ +; RUN: llc -debug-only=misched -march=amdgcn -verify-machineinstrs %s -o - 2>&1| FileCheck -check-prefix=SI-NOHSA -check-prefix=FUNC -check-prefix=DEBUG %s +; REQUIRES: asserts + +; Verify that the extload generated from %eval has the default +; alignment size (2) corresponding to the underlying memory size (i16) +; size and not 4 corresponding to the sign-extended size (i32). + +; DEBUG: {{^}}# Machine code for function extload_align: +; DEBUG: mem:LD2[<unknown>]{{[^(]}} +; DEBUG: {{^}}# End machine code for function extload_align. + +define void @extload_align(i32* %out, i32 %index) #0 { + %v0 = alloca [4 x i16] + %a1 = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 0 + %a2 = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 1 + store i16 0, i16* %a1 + store i16 1, i16* %a2 + %a = getelementptr inbounds [4 x i16], [4 x i16]* %v0, i32 0, i32 %index + %val = load i16, i16* %a + %eval = sext i16 %val to i32 + store i32 %eval, i32* %out + ret void +}
\ No newline at end of file diff --git a/test/CodeGen/AMDGPU/extload-private.ll b/test/CodeGen/AMDGPU/extload-private.ll index 3f27370d7037..6cebe5f495c5 100644 --- a/test/CodeGen/AMDGPU/extload-private.ll +++ b/test/CodeGen/AMDGPU/extload-private.ll @@ -2,7 +2,7 @@ ; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}load_i8_sext_private: -; SI: buffer_load_sbyte v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen +; SI: buffer_load_sbyte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+$}} define void @load_i8_sext_private(i32 addrspace(1)* %out) { entry: %tmp0 = alloca i8 @@ -13,7 +13,7 @@ entry: } ; FUNC-LABEL: {{^}}load_i8_zext_private: -; SI: buffer_load_ubyte v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen +; SI: buffer_load_ubyte v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+$}} define void @load_i8_zext_private(i32 addrspace(1)* %out) { entry: %tmp0 = alloca i8 @@ -24,7 +24,7 @@ entry: } ; FUNC-LABEL: {{^}}load_i16_sext_private: -; SI: buffer_load_sshort v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen +; SI: buffer_load_sshort v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+$}} define void @load_i16_sext_private(i32 addrspace(1)* %out) { entry: %tmp0 = alloca i16 @@ -35,7 +35,7 @@ entry: } ; FUNC-LABEL: {{^}}load_i16_zext_private: -; SI: buffer_load_ushort v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen +; SI: buffer_load_ushort v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+$}} define void @load_i16_zext_private(i32 addrspace(1)* %out) { entry: %tmp0 = alloca i16 diff --git a/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll b/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll index e012cb07163b..eea44b8a006c 100644 --- a/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll +++ b/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll @@ -1,7 +1,7 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s -; FUNC-LABEL: {{^}}extract_vector_elt_v2i16: +; GCN-LABEL: {{^}}extract_vector_elt_v2i16: ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort ; GCN: buffer_store_short @@ -15,7 +15,7 @@ define void @extract_vector_elt_v2i16(i16 addrspace(1)* %out, <2 x i16> %foo) #0 ret void } -; FUNC-LABEL: {{^}}extract_vector_elt_v3i16: +; GCN-LABEL: {{^}}extract_vector_elt_v3i16: ; GCN: buffer_load_ushort ; GCN: buffer_store_short ; GCN: buffer_store_short @@ -28,7 +28,7 @@ define void @extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> %foo) #0 ret void } -; FUNC-LABEL: {{^}}extract_vector_elt_v4i16: +; GCN-LABEL: {{^}}extract_vector_elt_v4i16: ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort ; GCN: buffer_store_short @@ -42,8 +42,7 @@ define void @extract_vector_elt_v4i16(i16 addrspace(1)* %out, <4 x i16> %foo) #0 ret void } - -; FUNC-LABEL: {{^}}dynamic_extract_vector_elt_v3i16: +; GCN-LABEL: {{^}}dynamic_extract_vector_elt_v3i16: ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort @@ -52,7 +51,6 @@ define void @extract_vector_elt_v4i16(i16 addrspace(1)* %out, <4 x i16> %foo) #0 ; GCN: buffer_store_short ; GCN: buffer_store_short -; GCN: buffer_store_short ; GCN: buffer_load_ushort ; GCN: buffer_store_short define void @dynamic_extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> %foo, i32 %idx) #0 { @@ -62,7 +60,7 @@ define void @dynamic_extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> ret void } -; FUNC-LABEL: {{^}}dynamic_extract_vector_elt_v4i16: +; GCN-LABEL: {{^}}dynamic_extract_vector_elt_v4i16: ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort @@ -73,7 +71,6 @@ define void @dynamic_extract_vector_elt_v3i16(i16 addrspace(1)* %out, <3 x i16> ; GCN: buffer_store_short ; GCN: buffer_store_short -; GCN: buffer_store_short ; GCN: buffer_load_ushort ; GCN: buffer_store_short define void @dynamic_extract_vector_elt_v4i16(i16 addrspace(1)* %out, <4 x i16> %foo, i32 %idx) #0 { diff --git a/test/CodeGen/AMDGPU/fabs.f16.ll b/test/CodeGen/AMDGPU/fabs.f16.ll new file mode 100644 index 000000000000..c64aa6228c71 --- /dev/null +++ b/test/CodeGen/AMDGPU/fabs.f16.ll @@ -0,0 +1,93 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; DAGCombiner will transform: +; (fabs (f16 bitcast (i16 a))) => (f16 bitcast (and (i16 a), 0x7FFFFFFF)) +; unless isFabsFree returns true + +; GCN-LABEL: {{^}}fabs_free_f16: +; GCN: flat_load_ushort [[VAL:v[0-9]+]], +; GCN: v_and_b32_e32 [[RESULT:v[0-9]+]], 0x7fff, [[VAL]] +; GCN: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] + +define void @fabs_free_f16(half addrspace(1)* %out, i16 %in) { + %bc= bitcast i16 %in to half + %fabs = call half @llvm.fabs.f16(half %bc) + store half %fabs, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fabs_f16: +; CI: flat_load_ushort [[VAL:v[0-9]+]], +; CI: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[VAL]] +; CI: v_cvt_f16_f32_e64 [[RESULT:v[0-9]+]], |[[CVT0]]| +; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fabs_f16(half addrspace(1)* %out, half %in) { + %fabs = call half @llvm.fabs.f16(half %in) + store half %fabs, half addrspace(1)* %out + ret void +} + +; FIXME: Should be able to use single and +; GCN-LABEL: {{^}}fabs_v2f16: +; CI: v_cvt_f32_f16_e32 v{{[0-9]+}}, +; CI: v_cvt_f32_f16_e32 v{{[0-9]+}}, +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| + +; VI: flat_load_ushort [[LO:v[0-9]+]] +; VI: flat_load_ushort [[HI:v[0-9]+]] +; VI: v_mov_b32_e32 [[MASK:v[0-9]+]], 0x7fff{{$}} +; VI-DAG: v_and_b32_e32 [[FABS_LO:v[0-9]+]], [[MASK]], [[LO]] +; VI-DAG: v_and_b32_e32 [[FABS_LO:v[0-9]+]], [[MASK]], [[HI]] +; VI-DAG: v_lshlrev_b32_e32 v{{[0-9]+}}, 16, +; VI-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xffff, +; VI: v_or_b32 +; VI: flat_store_dword +define void @fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half> %in) { + %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %in) + store <2 x half> %fabs, <2 x half> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fabs_v4f16: +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}| + +; VI: v_mov_b32_e32 [[MASK:v[0-9]+]], 0x7fff{{$}} +; VI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}} +; VI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}} +; VI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}} +; VI: v_and_b32_e32 v{{[0-9]+}}, [[MASK]], v{{[0-9]+}} + +; GCN: flat_store_dwordx2 +define void @fabs_v4f16(<4 x half> addrspace(1)* %out, <4 x half> %in) { + %fabs = call <4 x half> @llvm.fabs.v4f16(<4 x half> %in) + store <4 x half> %fabs, <4 x half> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fabs_fold_f16: +; GCN: flat_load_ushort [[IN0:v[0-9]+]] +; GCN: flat_load_ushort [[IN1:v[0-9]+]] +; CI-DAG: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[IN0]] +; CI-DAG: v_cvt_f32_f16_e32 [[CVT1:v[0-9]+]], [[IN1]] +; CI: v_mul_f32_e64 [[RESULT:v[0-9]+]], |[[CVT1]]|, [[CVT0]] +; CI: v_cvt_f16_f32_e32 [[CVTRESULT:v[0-9]+]], [[RESULT]] +; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[CVTRESULT]] + +; VI-NOT: and +; VI: v_mul_f16_e64 [[RESULT:v[0-9]+]], |[[IN1]]|, [[IN0]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fabs_fold_f16(half addrspace(1)* %out, half %in0, half %in1) { + %fabs = call half @llvm.fabs.f16(half %in0) + %fmul = fmul half %fabs, %in1 + store half %fmul, half addrspace(1)* %out + ret void +} + +declare half @llvm.fabs.f16(half) readnone +declare <2 x half> @llvm.fabs.v2f16(<2 x half>) readnone +declare <4 x half> @llvm.fabs.v4f16(<4 x half>) readnone diff --git a/test/CodeGen/AMDGPU/fabs.f64.ll b/test/CodeGen/AMDGPU/fabs.f64.ll index db8093047a36..f7780b875ff5 100644 --- a/test/CodeGen/AMDGPU/fabs.f64.ll +++ b/test/CodeGen/AMDGPU/fabs.f64.ll @@ -55,7 +55,7 @@ define void @fabs_v4f64(<4 x double> addrspace(1)* %out, <4 x double> %in) { ; SI-LABEL: {{^}}fabs_fold_f64: ; SI: s_load_dwordx2 [[ABS_VALUE:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; SI-NOT: and -; SI: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, |[[ABS_VALUE]]|, {{v\[[0-9]+:[0-9]+\]}} +; SI: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, |[[ABS_VALUE]]| ; SI: s_endpgm define void @fabs_fold_f64(double addrspace(1)* %out, double %in0, double %in1) { %fabs = call double @llvm.fabs.f64(double %in0) @@ -67,7 +67,7 @@ define void @fabs_fold_f64(double addrspace(1)* %out, double %in0, double %in1) ; SI-LABEL: {{^}}fabs_fn_fold_f64: ; SI: s_load_dwordx2 [[ABS_VALUE:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; SI-NOT: and -; SI: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, |[[ABS_VALUE]]|, {{v\[[0-9]+:[0-9]+\]}} +; SI: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, |[[ABS_VALUE]]| ; SI: s_endpgm define void @fabs_fn_fold_f64(double addrspace(1)* %out, double %in0, double %in1) { %fabs = call double @fabs(double %in0) diff --git a/test/CodeGen/AMDGPU/fabs.ll b/test/CodeGen/AMDGPU/fabs.ll index 419a73d02669..3742b58d500a 100644 --- a/test/CodeGen/AMDGPU/fabs.ll +++ b/test/CodeGen/AMDGPU/fabs.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s @@ -75,7 +75,7 @@ define void @fabs_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) { ; SI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb ; VI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c ; GCN-NOT: and -; GCN: v_mul_f32_e64 v{{[0-9]+}}, |[[ABS_VALUE]]|, v{{[0-9]+}} +; GCN: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, |[[ABS_VALUE]]| define void @fabs_fn_fold(float addrspace(1)* %out, float %in0, float %in1) { %fabs = call float @fabs(float %in0) %fmul = fmul float %fabs, %in1 @@ -87,7 +87,7 @@ define void @fabs_fn_fold(float addrspace(1)* %out, float %in0, float %in1) { ; SI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb ; VI: s_load_dword [[ABS_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c ; GCN-NOT: and -; GCN: v_mul_f32_e64 v{{[0-9]+}}, |[[ABS_VALUE]]|, v{{[0-9]+}} +; GCN: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, |[[ABS_VALUE]]| define void @fabs_fold(float addrspace(1)* %out, float %in0, float %in1) { %fabs = call float @llvm.fabs.f32(float %in0) %fmul = fmul float %fabs, %in1 diff --git a/test/CodeGen/AMDGPU/fadd.f16.ll b/test/CodeGen/AMDGPU/fadd.f16.ll new file mode 100644 index 000000000000..fb2d418b4436 --- /dev/null +++ b/test/CodeGen/AMDGPU/fadd.f16.ll @@ -0,0 +1,150 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fadd_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_add_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_add_f16_e32 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fadd_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fadd half %a.val, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fadd_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_add_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_add_f16_e32 v[[R_F16:[0-9]+]], 1.0, v[[B_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fadd_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = fadd half 1.0, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fadd_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4000{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_add_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_add_f16_e32 v[[R_F16:[0-9]+]], 2.0, v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fadd_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fadd half %a.val, 2.0 + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fadd_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_add_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_add_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_add_f16_e32 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_add_f16_e32 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fadd_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fadd <2 x half> %a.val, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fadd_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], 0x4000{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_add_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_add_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_add_f16_e32 v[[R_F16_0:[0-9]+]], 1.0, v[[B_V2_F16]] +; VI: v_add_f16_e32 v[[R_F16_1:[0-9]+]], 2.0, v[[B_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fadd_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fadd <2 x half> <half 1.0, half 2.0>, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fadd_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], 0x4000{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_add_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_add_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_add_f16_e32 v[[R_F16_0:[0-9]+]], 2.0, v[[A_V2_F16]] +; VI: v_add_f16_e32 v[[R_F16_1:[0-9]+]], 1.0, v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fadd_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fadd <2 x half> %a.val, <half 2.0, half 1.0> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/test/CodeGen/AMDGPU/fcanonicalize.f16.ll new file mode 100644 index 000000000000..fb693de0e39d --- /dev/null +++ b/test/CodeGen/AMDGPU/fcanonicalize.f16.ll @@ -0,0 +1,172 @@ +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare half @llvm.canonicalize.f16(half) #0 + +; GCN-LABEL: {{^}}v_test_canonicalize_var_f16: +; GCN: v_mul_f16_e32 [[REG:v[0-9]+]], 1.0, {{v[0-9]+}} +; GCN: buffer_store_short [[REG]] +define void @v_test_canonicalize_var_f16(half addrspace(1)* %out) #1 { + %val = load half, half addrspace(1)* %out + %canonicalized = call half @llvm.canonicalize.f16(half %val) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}s_test_canonicalize_var_f16: +; GCN: v_mul_f16_e64 [[REG:v[0-9]+]], 1.0, {{s[0-9]+}} +; GCN: buffer_store_short [[REG]] +define void @s_test_canonicalize_var_f16(half addrspace(1)* %out, i16 zeroext %val.arg) #1 { + %val = bitcast i16 %val.arg to half + %canonicalized = call half @llvm.canonicalize.f16(half %val) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_p0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_p0_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0.0) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_n0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffff8000{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_n0_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half -0.0) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_p1_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3c00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_p1_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 1.0) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_n1_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffbc00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_n1_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half -1.0) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_literal_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x4c00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_literal_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 16.0) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_no_denormals_fold_canonicalize_denormal0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_no_denormals_fold_canonicalize_denormal0_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH03FF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_denormals_fold_canonicalize_denormal0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3ff{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_denormals_fold_canonicalize_denormal0_f16(half addrspace(1)* %out) #3 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH03FF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_no_denormals_fold_canonicalize_denormal1_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_no_denormals_fold_canonicalize_denormal1_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH83FF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_denormals_fold_canonicalize_denormal1_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffff83ff{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_denormals_fold_canonicalize_denormal1_f16(half addrspace(1)* %out) #3 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH83FF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_qnan_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7c00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_qnan_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH7C00) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_qnan_value_neg1_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_qnan_value_neg1_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half bitcast (i16 -1 to half)) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_qnan_value_neg2_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_qnan_value_neg2_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half bitcast (i16 -2 to half)) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_snan0_value_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_snan0_value_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH7C01) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_snan1_value_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_snan1_value_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xH7DFF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_snan2_value_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_snan2_value_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xHFDFF) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}test_fold_canonicalize_snan3_value_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x7e00{{$}} +; GCN: buffer_store_short [[REG]] +define void @test_fold_canonicalize_snan3_value_f16(half addrspace(1)* %out) #1 { + %canonicalized = call half @llvm.canonicalize.f16(half 0xHFC01) + store half %canonicalized, half addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } +attributes #2 = { nounwind "target-features"="-fp16-denormals,-fp16-denormals" } +attributes #3 = { nounwind "target-features"="+fp16-denormals,+fp64-denormals" } diff --git a/test/CodeGen/AMDGPU/fceil64.ll b/test/CodeGen/AMDGPU/fceil64.ll index fb5853b808e4..98448db5dd24 100644 --- a/test/CodeGen/AMDGPU/fceil64.ll +++ b/test/CodeGen/AMDGPU/fceil64.ll @@ -13,8 +13,10 @@ declare <16 x double> @llvm.ceil.v16f64(<16 x double>) nounwind readnone ; CI: v_ceil_f64_e32 ; SI: s_bfe_u32 [[SEXP:s[0-9]+]], {{s[0-9]+}}, 0xb0014 ; SI-DAG: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x80000000 -; SI-DAG: s_add_i32 [[SEXP1:s[0-9]+]], [[SEXP]], 0xfffffc01 -; SI-DAG: s_lshr_b64 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], [[SEXP1]] +; FIXME: We should be using s_addk_i32 here, but the reg allocation hints +; are not always followed. +; SI-DAG: s_add_i32 [[SEXP0:s[0-9]+]], [[SEXP]], 0xfffffc01 +; SI-DAG: s_lshr_b64 s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], [[SEXP0]] ; SI-DAG: s_not_b64 ; SI-DAG: s_and_b64 ; SI-DAG: cmp_gt_i32 @@ -23,7 +25,7 @@ declare <16 x double> @llvm.ceil.v16f64(<16 x double>) nounwind readnone ; SI-DAG: cmp_lt_i32 ; SI-DAG: cndmask_b32 ; SI-DAG: cndmask_b32 -; SI-DAG: v_cmp_lt_f64 +; SI-DAG: v_cmp_gt_f64 ; SI-DAG: v_cmp_lg_f64 ; SI-DAG: v_cndmask_b32 ; SI: v_cndmask_b32 diff --git a/test/CodeGen/AMDGPU/fcmp.f16.ll b/test/CodeGen/AMDGPU/fcmp.f16.ll new file mode 100644 index 000000000000..c6e1ed5a1f29 --- /dev/null +++ b/test/CodeGen/AMDGPU/fcmp.f16.ll @@ -0,0 +1,777 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fcmp_f16_lt +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_lt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_lt( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp olt half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_lt_abs: +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] + +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] + +; SI: v_cmp_lt_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, |v[[A_F32]]|, |v[[B_F32]]| +; VI: v_cmp_lt_f16_e64 s{{\[[0-9]+:[0-9]+\]}}, |v[[A_F16]]|, |v[[B_F16]]| + +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_lt_abs( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %a.abs = call half @llvm.fabs.f16(half %a.val) + %b.abs = call half @llvm.fabs.f16(half %b.val) + %r.val = fcmp olt half %a.abs, %b.abs + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_eq +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_eq_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_eq_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_eq( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp oeq half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_le +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_le_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_le_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_le( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ole half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_gt +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_gt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_gt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_gt( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ogt half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_lg +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_lg_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_lg_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_lg( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp one half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_ge +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_ge_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_ge_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_ge( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp oge half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_o +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_o_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_o_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_o( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ord half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_u +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_u_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_u_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_u( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp uno half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_nge +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_nge_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_nge_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_nge( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ult half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_nlg +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_nlg_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_nlg_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_nlg( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ueq half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_ngt +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_ngt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_ngt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_ngt( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ule half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_nle +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_nle_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_nle_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_nle( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp ugt half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_neq +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_neq_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_neq_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_neq( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp une half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_f16_nlt +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_nlt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; VI: v_cmp_nlt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fcmp_f16_nlt( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fcmp uge half %a.val, %b.val + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_lt +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_lt_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_lt_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_lt_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_lt_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_lt( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp olt <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_eq +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_eq_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_eq_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_eq_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_eq_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_eq( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp oeq <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_le +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_le_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_le_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_le_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_le_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_le( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ole <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_gt +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_gt_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_gt_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_gt_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_gt_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_gt( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ogt <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_lg +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_lg_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_lg_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_lg_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_lg_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_lg( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp one <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_ge +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_ge_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_ge_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_ge_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_ge_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_ge( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp oge <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_o +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_o_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_o_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_o_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_o_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_o( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ord <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_u +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_u_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_u_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_u_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_u_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_u( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp uno <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_nge +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_nge_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_nge_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_nge_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_nge_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_nge( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ult <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_nlg +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_nlg_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_nlg_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_nlg_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_nlg_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_nlg( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ueq <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_ngt +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_ngt_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_ngt_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_ngt_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_ngt_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_ngt( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ule <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_nle +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_nle_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_nle_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_nle_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_nle_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_nle( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp ugt <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_neq +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_neq_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_neq_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_neq_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_neq_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_neq( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp une <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fcmp_v2f16_nlt +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cmp_nlt_f32_e32 vcc, v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cmp_nlt_f32_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F32_1]], v[[B_F32_1]] +; VI: v_cmp_nlt_f16_e32 vcc, v[[A_V2_F16]], v[[B_V2_F16]] +; VI: v_cmp_nlt_f16_e64 s[{{[0-9]+}}:{{[0-9]+}}], v[[A_F16_1]], v[[B_F16_1]] +; GCN: v_cndmask_b32_e64 v[[R_I32_0:[0-9]+]] +; GCN: v_cndmask_b32_e64 v[[R_I32_1:[0-9]+]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_I32_0]]:[[R_I32_1]]{{\]}} +; GCN: s_endpgm +define void @fcmp_v2f16_nlt( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fcmp uge <2 x half> %a.val, %b.val + %r.val.sext = sext <2 x i1> %r.val to <2 x i32> + store <2 x i32> %r.val.sext, <2 x i32> addrspace(1)* %r + ret void +} + +declare half @llvm.fabs.f16(half) #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fcopysign.f32.ll b/test/CodeGen/AMDGPU/fcopysign.f32.ll index b719d5a39785..9f01a660be3a 100644 --- a/test/CodeGen/AMDGPU/fcopysign.f32.ll +++ b/test/CodeGen/AMDGPU/fcopysign.f32.ll @@ -1,8 +1,7 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s - declare float @llvm.copysign.f32(float, float) nounwind readnone declare <2 x float> @llvm.copysign.v2f32(<2 x float>, <2 x float>) nounwind readnone declare <4 x float> @llvm.copysign.v4f32(<4 x float>, <4 x float>) nounwind readnone @@ -15,7 +14,7 @@ declare <4 x float> @llvm.copysign.v4f32(<4 x float>, <4 x float>) nounwind read ; VI: s_load_dword [[SSIGN:s[0-9]+]], {{.*}} 0x30 ; GCN-DAG: v_mov_b32_e32 [[VSIGN:v[0-9]+]], [[SSIGN]] ; GCN-DAG: v_mov_b32_e32 [[VMAG:v[0-9]+]], [[SMAG]] -; GCN-DAG: s_mov_b32 [[SCONST:s[0-9]+]], 0x7fffffff +; GCN-DAG: s_brev_b32 [[SCONST:s[0-9]+]], -2 ; GCN: v_bfi_b32 [[RESULT:v[0-9]+]], [[SCONST]], [[VMAG]], [[VSIGN]] ; GCN: buffer_store_dword [[RESULT]], ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/fcopysign.f64.ll b/test/CodeGen/AMDGPU/fcopysign.f64.ll index 738a35fb3b89..b34a4695387f 100644 --- a/test/CodeGen/AMDGPU/fcopysign.f64.ll +++ b/test/CodeGen/AMDGPU/fcopysign.f64.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s declare double @llvm.copysign.f64(double, double) nounwind readnone @@ -12,7 +12,7 @@ declare <4 x double> @llvm.copysign.v4f64(<4 x double>, <4 x double>) nounwind r ; VI-DAG: s_load_dwordx2 s{{\[}}[[SSIGN_LO:[0-9]+]]:[[SSIGN_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x34 ; GCN-DAG: v_mov_b32_e32 v[[VSIGN_HI:[0-9]+]], s[[SSIGN_HI]] ; GCN-DAG: v_mov_b32_e32 v[[VMAG_HI:[0-9]+]], s[[SMAG_HI]] -; GCN-DAG: s_mov_b32 [[SCONST:s[0-9]+]], 0x7fffffff +; GCN-DAG: s_brev_b32 [[SCONST:s[0-9]+]], -2 ; GCN-DAG: v_bfi_b32 v[[VRESULT_HI:[0-9]+]], [[SCONST]], v[[VMAG_HI]], v[[VSIGN_HI]] ; GCN-DAG: v_mov_b32_e32 v[[VMAG_LO:[0-9]+]], s[[SMAG_LO]] ; GCN: buffer_store_dwordx2 v{{\[}}[[VMAG_LO]]:[[VRESULT_HI]]{{\]}} @@ -23,6 +23,22 @@ define void @test_copysign_f64(double addrspace(1)* %out, double %mag, double %s ret void } +; FUNC-LABEL: {{^}}test_copysign_f64_f32: +; GCN-DAG: s_load_dwordx2 s{{\[}}[[SMAG_LO:[0-9]+]]:[[SMAG_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}} +; GCN-DAG: s_load_dword s[[SSIGN:[0-9]+]], s{{\[[0-9]+:[0-9]+\]}} +; GCN-DAG: s_brev_b32 [[SCONST:s[0-9]+]], -2{{$}} +; GCN-DAG: v_mov_b32_e32 v[[VMAG_HI:[0-9]+]], s[[SMAG_HI]] +; GCN-DAG: v_mov_b32_e32 v[[VSIGN:[0-9]+]], s[[SSIGN]] +; GCN-DAG: v_bfi_b32 v[[VRESULT_HI:[0-9]+]], [[SCONST]], v[[VMAG_HI]], v[[VSIGN]] +; GCN-DAG: v_mov_b32_e32 v[[VMAG_LO:[0-9]+]], s[[SMAG_LO]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VMAG_LO]]:[[VRESULT_HI]]{{\]}} +define void @test_copysign_f64_f32(double addrspace(1)* %out, double %mag, float %sign) nounwind { + %c = fpext float %sign to double + %result = call double @llvm.copysign.f64(double %mag, double %c) + store double %result, double addrspace(1)* %out, align 8 + ret void +} + ; FUNC-LABEL: {{^}}test_copysign_v2f64: ; GCN: s_endpgm define void @test_copysign_v2f64(<2 x double> addrspace(1)* %out, <2 x double> %mag, <2 x double> %sign) nounwind { diff --git a/test/CodeGen/AMDGPU/fdiv.f16.ll b/test/CodeGen/AMDGPU/fdiv.f16.ll new file mode 100644 index 000000000000..fad45c6816d2 --- /dev/null +++ b/test/CodeGen/AMDGPU/fdiv.f16.ll @@ -0,0 +1,213 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; Make sure fdiv is promoted to f32. + +; GCN-LABEL: {{^}}v_fdiv_f16 +; SI: v_cvt_f32_f16 +; SI: v_cvt_f32_f16 +; SI: v_div_scale_f32 +; SI-DAG: v_div_scale_f32 +; SI-DAG: v_rcp_f32 +; SI: v_fma_f32 +; SI: v_fma_f32 +; SI: v_mul_f32 +; SI: v_fma_f32 +; SI: v_fma_f32 +; SI: v_fma_f32 +; SI: v_div_fmas_f32 +; SI: v_div_fixup_f32 +; SI: v_cvt_f16_f32 + +; VI: flat_load_ushort [[LHS:v[0-9]+]] +; VI: flat_load_ushort [[RHS:v[0-9]+]] + +; VI-DAG: v_cvt_f32_f16_e32 [[CVT_LHS:v[0-9]+]], [[LHS]] +; VI-DAG: v_cvt_f32_f16_e32 [[CVT_RHS:v[0-9]+]], [[RHS]] + +; VI-DAG: v_rcp_f32_e32 [[RCP_RHS:v[0-9]+]], [[CVT_RHS]] +; VI: v_mul_f32_e32 [[MUL:v[0-9]+]], [[RCP_RHS]], [[CVT_LHS]] +; VI: v_cvt_f16_f32_e32 [[CVT_BACK:v[0-9]+]], [[MUL]] +; VI: v_div_fixup_f16 [[RESULT:v[0-9]+]], [[CVT_BACK]], [[RHS]], [[LHS]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_fdiv_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.a = getelementptr inbounds half, half addrspace(1)* %a, i64 %tid.ext + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %a.val = load volatile half, half addrspace(1)* %gep.a + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv half %a.val, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rcp_f16: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_rcp_f16_e32 [[RESULT:v[0-9]+]], [[VAL]] +; VI-NOT: [[RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rcp_f16(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv half 1.0, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rcp_f16_abs: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_rcp_f16_e64 [[RESULT:v[0-9]+]], |[[VAL]]| +; VI-NOT: [RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rcp_f16_abs(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %b.abs = call half @llvm.fabs.f16(half %b.val) + %r.val = fdiv half 1.0, %b.abs + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rcp_f16_arcp: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_rcp_f16_e32 [[RESULT:v[0-9]+]], [[VAL]] +; VI-NOT: [[RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rcp_f16_arcp(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv arcp half 1.0, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rcp_f16_neg: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_rcp_f16_e64 [[RESULT:v[0-9]+]], -[[VAL]] +; VI-NOT: [RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rcp_f16_neg(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv half -1.0, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rsq_f16: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_rsq_f16_e32 [[RESULT:v[0-9]+]], [[VAL]] +; VI-NOT: [RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rsq_f16(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %b.sqrt = call half @llvm.sqrt.f16(half %b.val) + %r.val = fdiv half 1.0, %b.sqrt + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_rsq_f16_neg: +; VI: flat_load_ushort [[VAL:v[0-9]+]] +; VI-NOT: [[VAL]] +; VI: v_sqrt_f16_e32 [[SQRT:v[0-9]+]], [[VAL]] +; VI-NEXT: v_rcp_f16_e64 [[RESULT:v[0-9]+]], -[[SQRT]] +; VI-NOT: [RESULT]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_rsq_f16_neg(half addrspace(1)* %r, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %b.val = load volatile half, half addrspace(1)* %gep.b + %b.sqrt = call half @llvm.sqrt.f16(half %b.val) + %r.val = fdiv half -1.0, %b.sqrt + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_fdiv_f16_arcp: +; VI: flat_load_ushort [[LHS:v[0-9]+]] +; VI: flat_load_ushort [[RHS:v[0-9]+]] + +; VI: v_rcp_f16_e32 [[RCP:v[0-9]+]], [[RHS]] +; VI: v_mul_f16_e32 [[RESULT:v[0-9]+]], [[RCP]], [[LHS]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_fdiv_f16_arcp(half addrspace(1)* %r, half addrspace(1)* %a, half addrspace(1)* %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.a = getelementptr inbounds half, half addrspace(1)* %a, i64 %tid.ext + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %a.val = load volatile half, half addrspace(1)* %gep.a + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv arcp half %a.val, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +; GCN-LABEL: {{^}}v_fdiv_f16_unsafe: +; VI: flat_load_ushort [[LHS:v[0-9]+]] +; VI: flat_load_ushort [[RHS:v[0-9]+]] + +; VI: v_rcp_f16_e32 [[RCP:v[0-9]+]], [[RHS]] +; VI: v_mul_f16_e32 [[RESULT:v[0-9]+]], [[RCP]], [[LHS]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @v_fdiv_f16_unsafe(half addrspace(1)* %r, half addrspace(1)* %a, half addrspace(1)* %b) #2 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %gep.a = getelementptr inbounds half, half addrspace(1)* %a, i64 %tid.ext + %gep.b = getelementptr inbounds half, half addrspace(1)* %b, i64 %tid.ext + %gep.r = getelementptr inbounds half, half addrspace(1)* %r, i64 %tid.ext + %a.val = load volatile half, half addrspace(1)* %gep.a + %b.val = load volatile half, half addrspace(1)* %gep.b + %r.val = fdiv half %a.val, %b.val + store half %r.val, half addrspace(1)* %gep.r + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 +declare half @llvm.sqrt.f16(half) #1 +declare half @llvm.fabs.f16(half) #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind "unsafe-fp-math"="true" } diff --git a/test/CodeGen/AMDGPU/fdiv.f64.ll b/test/CodeGen/AMDGPU/fdiv.f64.ll index 3343b681b9fe..2ace0762c677 100644 --- a/test/CodeGen/AMDGPU/fdiv.f64.ll +++ b/test/CodeGen/AMDGPU/fdiv.f64.ll @@ -15,8 +15,8 @@ ; COMMON-DAG: v_rcp_f64_e32 [[RCP_SCALE0:v\[[0-9]+:[0-9]+\]]], [[SCALE0]] -; SI-DAG: v_cmp_eq_i32_e32 vcc, {{v[0-9]+}}, {{v[0-9]+}} -; SI-DAG: v_cmp_eq_i32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, {{v[0-9]+}} +; SI-DAG: v_cmp_eq_u32_e32 vcc, {{v[0-9]+}}, {{v[0-9]+}} +; SI-DAG: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], {{v[0-9]+}}, {{v[0-9]+}} ; SI-DAG: s_xor_b64 vcc, [[CMP0]], vcc ; COMMON-DAG: v_fma_f64 [[FMA0:v\[[0-9]+:[0-9]+\]]], -[[SCALE0]], [[RCP_SCALE0]], 1.0 diff --git a/test/CodeGen/AMDGPU/fdiv.ll b/test/CodeGen/AMDGPU/fdiv.ll index 65464cdba604..a96035fca3dc 100644 --- a/test/CodeGen/AMDGPU/fdiv.ll +++ b/test/CodeGen/AMDGPU/fdiv.ll @@ -1,4 +1,6 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s ; These tests check that fdiv is expanded correctly and also test that the @@ -11,18 +13,20 @@ ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS -; SI: v_div_scale_f32 -; SI-DAG: v_div_scale_f32 +; GCN: v_div_scale_f32 [[NUM_SCALE:v[0-9]+]] +; GCN-DAG: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]] +; GCN-DAG: v_rcp_f32_e32 [[NUM_RCP:v[0-9]+]], [[NUM_SCALE]] -; SI-DAG: v_rcp_f32 -; SI: v_fma_f32 -; SI: v_fma_f32 -; SI: v_mul_f32 -; SI: v_fma_f32 -; SI: v_fma_f32 -; SI: v_fma_f32 -; SI: v_div_fmas_f32 -; SI: v_div_fixup_f32 +; GCN: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 +; GCN: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0 +; GCN: v_fma_f32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]], [[NUM_RCP]] +; GCN: v_mul_f32_e32 [[C:v[0-9]+]], [[B]], [[DEN_SCALE]] +; GCN: v_fma_f32 [[D:v[0-9]+]], -[[NUM_SCALE]], [[C]], [[DEN_SCALE]] +; GCN: v_fma_f32 [[E:v[0-9]+]], [[D]], [[B]], [[C]] +; GCN: v_fma_f32 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]], [[DEN_SCALE]] +; GCN: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 +; GCN: v_div_fmas_f32 [[FMAS:v[0-9]+]], [[F]], [[B]], [[E]] +; GCN: v_div_fixup_f32 v{{[0-9]+}}, [[FMAS]], define void @fdiv_f32(float addrspace(1)* %out, float %a, float %b) #0 { entry: %fdiv = fdiv float %a, %b @@ -30,12 +34,37 @@ entry: ret void } +; FUNC-LABEL: {{^}}fdiv_f32_denormals: +; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W +; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS + +; GCN: v_div_scale_f32 [[NUM_SCALE:v[0-9]+]] +; GCN-DAG: v_div_scale_f32 [[DEN_SCALE:v[0-9]+]] +; GCN-DAG: v_rcp_f32_e32 [[NUM_RCP:v[0-9]+]], [[NUM_SCALE]] + +; GCN-NOT: s_setreg +; GCN: v_fma_f32 [[A:v[0-9]+]], -[[NUM_SCALE]], [[NUM_RCP]], 1.0 +; GCN: v_fma_f32 [[B:v[0-9]+]], [[A]], [[NUM_RCP]], [[NUM_RCP]] +; GCN: v_mul_f32_e32 [[C:v[0-9]+]], [[B]], [[DEN_SCALE]] +; GCN: v_fma_f32 [[D:v[0-9]+]], -[[NUM_SCALE]], [[C]], [[DEN_SCALE]] +; GCN: v_fma_f32 [[E:v[0-9]+]], [[D]], [[B]], [[C]] +; GCN: v_fma_f32 [[F:v[0-9]+]], -[[NUM_SCALE]], [[E]], [[DEN_SCALE]] +; GCN-NOT: s_setreg +; GCN: v_div_fmas_f32 [[FMAS:v[0-9]+]], [[F]], [[B]], [[E]] +; GCN: v_div_fixup_f32 v{{[0-9]+}}, [[FMAS]], +define void @fdiv_f32_denormals(float addrspace(1)* %out, float %a, float %b) #2 { +entry: + %fdiv = fdiv float %a, %b + store float %fdiv, float addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}fdiv_25ulp_f32: -; SI: v_cndmask_b32 -; SI: v_mul_f32 -; SI: v_rcp_f32 -; SI: v_mul_f32 -; SI: v_mul_f32 +; GCN: v_cndmask_b32 +; GCN: v_mul_f32 +; GCN: v_rcp_f32 +; GCN: v_mul_f32 +; GCN: v_mul_f32 define void @fdiv_25ulp_f32(float addrspace(1)* %out, float %a, float %b) #0 { entry: %fdiv = fdiv float %a, %b, !fpmath !0 @@ -45,9 +74,9 @@ entry: ; Use correct fdiv ; FUNC-LABEL: {{^}}fdiv_25ulp_denormals_f32: -; SI: v_fma_f32 -; SI: v_div_fmas_f32 -; SI: v_div_fixup_f32 +; GCN: v_fma_f32 +; GCN: v_div_fmas_f32 +; GCN: v_div_fixup_f32 define void @fdiv_25ulp_denormals_f32(float addrspace(1)* %out, float %a, float %b) #2 { entry: %fdiv = fdiv float %a, %b, !fpmath !0 @@ -56,10 +85,10 @@ entry: } ; FUNC-LABEL: {{^}}fdiv_fast_denormals_f32: -; SI: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} -; SI: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] -; SI-NOT: [[RESULT]] -; SI: buffer_store_dword [[RESULT]] +; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} +; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] define void @fdiv_fast_denormals_f32(float addrspace(1)* %out, float %a, float %b) #2 { entry: %fdiv = fdiv fast float %a, %b @@ -71,10 +100,10 @@ entry: ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS -; SI: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} -; SI: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] -; SI-NOT: [[RESULT]] -; SI: buffer_store_dword [[RESULT]] +; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} +; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] define void @fdiv_f32_fast_math(float addrspace(1)* %out, float %a, float %b) #0 { entry: %fdiv = fdiv fast float %a, %b @@ -86,10 +115,10 @@ entry: ; R600-DAG: RECIP_IEEE * T{{[0-9]+\.[XYZW]}}, KC0[2].W ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z, PS -; SI: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} -; SI: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] -; SI-NOT: [[RESULT]] -; SI: buffer_store_dword [[RESULT]] +; GCN: v_rcp_f32_e32 [[RCP:v[0-9]+]], s{{[0-9]+}} +; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], s{{[0-9]+}}, [[RCP]] +; GCN-NOT: [[RESULT]] +; GCN: buffer_store_dword [[RESULT]] define void @fdiv_f32_arcp_math(float addrspace(1)* %out, float %a, float %b) #0 { entry: %fdiv = fdiv arcp float %a, %b @@ -103,10 +132,10 @@ entry: ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, PS -; SI: v_div_scale_f32 -; SI: v_div_scale_f32 -; SI: v_div_scale_f32 -; SI: v_div_scale_f32 +; GCN: v_div_scale_f32 +; GCN: v_div_scale_f32 +; GCN: v_div_scale_f32 +; GCN: v_div_scale_f32 define void @fdiv_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b) #0 { entry: %fdiv = fdiv <2 x float> %a, %b @@ -115,8 +144,8 @@ entry: } ; FUNC-LABEL: {{^}}fdiv_ulp25_v2f32: -; SI: v_cmp_gt_f32 -; SI: v_cmp_gt_f32 +; GCN: v_cmp_gt_f32 +; GCN: v_cmp_gt_f32 define void @fdiv_ulp25_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b) #0 { entry: %fdiv = fdiv arcp <2 x float> %a, %b, !fpmath !0 @@ -130,8 +159,8 @@ entry: ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, PS -; SI: v_rcp_f32 -; SI: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 define void @fdiv_v2f32_fast_math(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b) #0 { entry: %fdiv = fdiv fast <2 x float> %a, %b @@ -145,8 +174,8 @@ entry: ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[3].X, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW]}}, KC0[2].W, PS -; SI: v_rcp_f32 -; SI: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 define void @fdiv_v2f32_arcp_math(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b) #0 { entry: %fdiv = fdiv arcp <2 x float> %a, %b @@ -164,10 +193,10 @@ entry: ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS -; SI: v_div_fixup_f32 -; SI: v_div_fixup_f32 -; SI: v_div_fixup_f32 -; SI: v_div_fixup_f32 +; GCN: v_div_fixup_f32 +; GCN: v_div_fixup_f32 +; GCN: v_div_fixup_f32 +; GCN: v_div_fixup_f32 define void @fdiv_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) #0 { %b_ptr = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 1 %a = load <4 x float>, <4 x float> addrspace(1) * %in @@ -187,10 +216,10 @@ define void @fdiv_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1) ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS -; SI: v_rcp_f32 -; SI: v_rcp_f32 -; SI: v_rcp_f32 -; SI: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 define void @fdiv_v4f32_fast_math(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) #0 { %b_ptr = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 1 %a = load <4 x float>, <4 x float> addrspace(1) * %in @@ -210,10 +239,10 @@ define void @fdiv_v4f32_fast_math(<4 x float> addrspace(1)* %out, <4 x float> ad ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS ; R600-DAG: MUL_IEEE {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}}, PS -; SI: v_rcp_f32 -; SI: v_rcp_f32 -; SI: v_rcp_f32 -; SI: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 +; GCN: v_rcp_f32 define void @fdiv_v4f32_arcp_math(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) #0 { %b_ptr = getelementptr <4 x float>, <4 x float> addrspace(1)* %in, i32 1 %a = load <4 x float>, <4 x float> addrspace(1) * %in diff --git a/test/CodeGen/AMDGPU/ffloor.f64.ll b/test/CodeGen/AMDGPU/ffloor.f64.ll index ea708a2b7bbd..904dfe31b15e 100644 --- a/test/CodeGen/AMDGPU/ffloor.f64.ll +++ b/test/CodeGen/AMDGPU/ffloor.f64.ll @@ -14,9 +14,9 @@ declare <16 x double> @llvm.floor.v16f64(<16 x double>) nounwind readnone ; CI: v_floor_f64_e32 ; SI: v_fract_f64_e32 ; SI-DAG: v_min_f64 -; SI-DAG: v_cmp_class_f64_e64 -; SI: v_cndmask_b32_e64 -; SI: v_cndmask_b32_e64 +; SI-DAG: v_cmp_class_f64_e64 vcc +; SI: v_cndmask_b32_e32 +; SI: v_cndmask_b32_e32 ; SI: v_add_f64 ; SI: s_endpgm define void @ffloor_f64(double addrspace(1)* %out, double %x) { @@ -29,9 +29,9 @@ define void @ffloor_f64(double addrspace(1)* %out, double %x) { ; CI: v_floor_f64_e64 ; SI: v_fract_f64_e64 {{v[[0-9]+:[0-9]+]}}, -[[INPUT:s[[0-9]+:[0-9]+]]] ; SI-DAG: v_min_f64 -; SI-DAG: v_cmp_class_f64_e64 -; SI: v_cndmask_b32_e64 -; SI: v_cndmask_b32_e64 +; SI-DAG: v_cmp_class_f64_e64 vcc +; SI: v_cndmask_b32_e32 +; SI: v_cndmask_b32_e32 ; SI: v_add_f64 {{v[[0-9]+:[0-9]+]}}, -[[INPUT]] ; SI: s_endpgm define void @ffloor_f64_neg(double addrspace(1)* %out, double %x) { @@ -45,9 +45,9 @@ define void @ffloor_f64_neg(double addrspace(1)* %out, double %x) { ; CI: v_floor_f64_e64 ; SI: v_fract_f64_e64 {{v[[0-9]+:[0-9]+]}}, -|[[INPUT:s[[0-9]+:[0-9]+]]]| ; SI-DAG: v_min_f64 -; SI-DAG: v_cmp_class_f64_e64 -; SI: v_cndmask_b32_e64 -; SI: v_cndmask_b32_e64 +; SI-DAG: v_cmp_class_f64_e64 vcc +; SI: v_cndmask_b32_e32 +; SI: v_cndmask_b32_e32 ; SI: v_add_f64 {{v[[0-9]+:[0-9]+]}}, -|[[INPUT]]| ; SI: s_endpgm define void @ffloor_f64_neg_abs(double addrspace(1)* %out, double %x) { diff --git a/test/CodeGen/AMDGPU/flat-address-space.ll b/test/CodeGen/AMDGPU/flat-address-space.ll index 5ca57fd3d350..0cfe6888b335 100644 --- a/test/CodeGen/AMDGPU/flat-address-space.ll +++ b/test/CodeGen/AMDGPU/flat-address-space.ll @@ -1,7 +1,6 @@ -; RUN: llc -O0 -march=amdgcn -mcpu=bonaire -mattr=-promote-alloca < %s | FileCheck -check-prefix=CHECK -check-prefix=CHECK-NO-PROMOTE %s -; RUN: llc -O0 -march=amdgcn -mcpu=bonaire -mattr=+promote-alloca < %s | FileCheck -check-prefix=CHECK -check-prefix=CHECK-PROMOTE %s -; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=-promote-alloca < %s | FileCheck -check-prefix=CHECK -check-prefix=CHECK-NO-PROMOTE %s -; RUN: llc -O0 -march=amdgcn -mcpu=tonga -mattr=+promote-alloca < %s | FileCheck -check-prefix=CHECK -check-prefix=CHECK-PROMOTE %s +; RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire < %s | FileCheck %s +; RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga < %s | FileCheck %s +; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -check-prefixes=CHECK,HSA %s ; Disable optimizations in case there are optimizations added that ; specialize away generic pointer accesses. @@ -73,7 +72,7 @@ define void @load_flat_i32(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noa ; CHECK: flat_load_dwordx2 define void @load_flat_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noalias %gptr) #0 { %fptr = addrspacecast i64 addrspace(1)* %gptr to i64 addrspace(4)* - %fload = load i64, i64 addrspace(4)* %fptr, align 4 + %fload = load i64, i64 addrspace(4)* %fptr, align 8 store i64 %fload, i64 addrspace(1)* %out, align 8 ret void } @@ -82,7 +81,7 @@ define void @load_flat_i64(i64 addrspace(1)* noalias %out, i64 addrspace(1)* noa ; CHECK: flat_load_dwordx4 define void @load_flat_v4i32(<4 x i32> addrspace(1)* noalias %out, <4 x i32> addrspace(1)* noalias %gptr) #0 { %fptr = addrspacecast <4 x i32> addrspace(1)* %gptr to <4 x i32> addrspace(4)* - %fload = load <4 x i32>, <4 x i32> addrspace(4)* %fptr, align 4 + %fload = load <4 x i32>, <4 x i32> addrspace(4)* %fptr, align 32 store <4 x i32> %fload, <4 x i32> addrspace(1)* %out, align 8 ret void } @@ -127,6 +126,52 @@ define void @zextload_flat_i16(i32 addrspace(1)* noalias %out, i16 addrspace(1)* ret void } +; CHECK-LABEL: flat_scratch_unaligned_load: +; CHECK: flat_load_ubyte +; CHECK: flat_load_ubyte +; CHECK: flat_load_ubyte +; CHECK: flat_load_ubyte +define void @flat_scratch_unaligned_load() { + %scratch = alloca i32 + %fptr = addrspacecast i32* %scratch to i32 addrspace(4)* + %ld = load volatile i32, i32 addrspace(4)* %fptr, align 1 + ret void +} + +; CHECK-LABEL: flat_scratch_unaligned_store: +; CHECK: flat_store_byte +; CHECK: flat_store_byte +; CHECK: flat_store_byte +; CHECK: flat_store_byte +define void @flat_scratch_unaligned_store() { + %scratch = alloca i32 + %fptr = addrspacecast i32* %scratch to i32 addrspace(4)* + store volatile i32 0, i32 addrspace(4)* %fptr, align 1 + ret void +} + +; CHECK-LABEL: flat_scratch_multidword_load: +; HSA: flat_load_dword +; HSA: flat_load_dword +; FIXME: These tests are broken for os = mesa3d, becasue it doesn't initialize flat_scr +define void @flat_scratch_multidword_load() { + %scratch = alloca <2 x i32> + %fptr = addrspacecast <2 x i32>* %scratch to <2 x i32> addrspace(4)* + %ld = load volatile <2 x i32>, <2 x i32> addrspace(4)* %fptr + ret void +} + +; CHECK-LABEL: flat_scratch_multidword_store: +; HSA: flat_store_dword +; HSA: flat_store_dword +; FIXME: These tests are broken for os = mesa3d, becasue it doesn't initialize flat_scr +define void @flat_scratch_multidword_store() { + %scratch = alloca <2 x i32> + %fptr = addrspacecast <2 x i32>* %scratch to <2 x i32> addrspace(4)* + store volatile <2 x i32> zeroinitializer, <2 x i32> addrspace(4)* %fptr + ret void +} + attributes #0 = { nounwind } attributes #1 = { nounwind convergent } attributes #3 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/flat-scratch-reg.ll b/test/CodeGen/AMDGPU/flat-scratch-reg.ll index b9489101f906..f15cbef56b19 100644 --- a/test/CodeGen/AMDGPU/flat-scratch-reg.ll +++ b/test/CodeGen/AMDGPU/flat-scratch-reg.ll @@ -1,10 +1,24 @@ -; RUN: llc < %s -march=amdgcn -mcpu=kaveri -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=CI --check-prefix=NO-XNACK -; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI --check-prefix=NO-XNACK -; RUN: llc < %s -march=amdgcn -mcpu=carrizo -mattr=+xnack -verify-machineinstrs | FileCheck %s --check-prefix=GCN --check-prefix=VI --check-prefix=XNACK +; RUN: llc -march=amdgcn -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK -check-prefix=GCN %s + +; RUN: llc -march=amdgcn -mcpu=carrizo -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=stoney -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=VI-NOXNACK -check-prefix=GCN %s + +; RUN: llc -march=amdgcn -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=VI-XNACK -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=stoney -verify-machineinstrs < %s | FileCheck -check-prefix=VI-XNACK -check-prefix=GCN %s + +; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-CI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=-xnack -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-VI-NOXNACK -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mtriple=amdgcn--amdhsa -mcpu=carrizo -mattr=+xnack -verify-machineinstrs < %s | FileCheck -check-prefix=HSA-VI-XNACK -check-prefix=GCN %s ; GCN-LABEL: {{^}}no_vcc_no_flat: -; NO-XNACK: ; NumSgprs: 8 -; XNACK: ; NumSgprs: 12 +; HSA-CI: is_xnack_enabled = 0 +; HSA-VI-NOXNACK: is_xnack_enabled = 0 +; HSA-VI-XNACK: is_xnack_enabled = 1 + +; CI: ; NumSgprs: 8 +; VI-NOXNACK: ; NumSgprs: 8 +; VI-XNACK: ; NumSgprs: 12 define void @no_vcc_no_flat() { entry: call void asm sideeffect "", "~{SGPR7}"() @@ -12,8 +26,13 @@ entry: } ; GCN-LABEL: {{^}}vcc_no_flat: -; NO-XNACK: ; NumSgprs: 10 -; XNACK: ; NumSgprs: 12 +; HSA-CI: is_xnack_enabled = 0 +; HSA-VI-NOXNACK: is_xnack_enabled = 0 +; HSA-VI-XNACK: is_xnack_enabled = 1 + +; CI: ; NumSgprs: 10 +; VI-NOXNACK: ; NumSgprs: 10 +; VI-XNACK: ; NumSgprs: 12 define void @vcc_no_flat() { entry: call void asm sideeffect "", "~{SGPR7},~{VCC}"() @@ -21,8 +40,16 @@ entry: } ; GCN-LABEL: {{^}}no_vcc_flat: -; CI: ; NumSgprs: 12 -; VI: ; NumSgprs: 14 +; HSA-CI: is_xnack_enabled = 0 +; HSA-VI-NOXNACK: is_xnack_enabled = 0 +; HSA-VI-XNACK: is_xnack_enabled = 1 + +; CI: ; NumSgprs: 8 +; VI-NOXNACK: ; NumSgprs: 8 +; VI-XNACK: ; NumSgprs: 12 +; HSA-CI: ; NumSgprs: 8 +; HSA-VI-NOXNACK: ; NumSgprs: 8 +; HSA-VI-XNACK: ; NumSgprs: 12 define void @no_vcc_flat() { entry: call void asm sideeffect "", "~{SGPR7},~{FLAT_SCR}"() @@ -30,8 +57,15 @@ entry: } ; GCN-LABEL: {{^}}vcc_flat: -; CI: ; NumSgprs: 12 -; VI: ; NumSgprs: 14 +; HSA-NOXNACK: is_xnack_enabled = 0 +; HSA-XNACK: is_xnack_enabled = 1 + +; CI: ; NumSgprs: 10 +; VI-NOXNACK: ; NumSgprs: 10 +; VI-XNACK: ; NumSgprs: 12 +; HSA-CI: ; NumSgprs: 10 +; HSA-VI-NOXNACK: ; NumSgprs: 10 +; HSA-VI-XNACK: ; NumSgprs: 12 define void @vcc_flat() { entry: call void asm sideeffect "", "~{SGPR7},~{VCC},~{FLAT_SCR}"() diff --git a/test/CodeGen/AMDGPU/fma-combine.ll b/test/CodeGen/AMDGPU/fma-combine.ll index 19deefe4d4a5..ab1b6d96a8d8 100644 --- a/test/CodeGen/AMDGPU/fma-combine.ll +++ b/test/CodeGen/AMDGPU/fma-combine.ll @@ -1,5 +1,10 @@ -; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-FASTFMAF -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-SLOWFMAF -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-NOFMA -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs -fp-contract=fast < %s | FileCheck -check-prefix=SI-NOFMA -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -mattr=+fp32-denormals < %s | FileCheck -check-prefix=SI-FMA -check-prefix=SI -check-prefix=FUNC %s + +; Note: The SI-FMA conversions of type x * (y + 1) --> x * y + x would be +; beneficial even without fp32 denormals, but they do require no-infs-fp-math +; for correctness. declare i32 @llvm.amdgcn.workitem.id.x() #0 declare double @llvm.fabs.f64(double) #0 @@ -369,7 +374,10 @@ define void @aggressive_combine_to_fma_fsub_1_f64(double addrspace(1)* noalias % ; ; FUNC-LABEL: {{^}}test_f32_mul_add_x_one_y: -; SI: v_mac_f32_e32 [[VY:v[0-9]]], [[VY:v[0-9]]], [[VX:v[0-9]]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_add_x_one_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -382,7 +390,10 @@ define void @test_f32_mul_add_x_one_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_add_x_one: -; SI: v_mac_f32_e32 [[VY:v[0-9]]], [[VY:v[0-9]]], [[VX:v[0-9]]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_y_add_x_one(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -395,7 +406,10 @@ define void @test_f32_mul_y_add_x_one(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_add_x_negone_y: -; SI: v_mad_f32 [[VX:v[0-9]]], [[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_add_x_negone_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -408,7 +422,10 @@ define void @test_f32_mul_add_x_negone_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_add_x_negone: -; SI: v_mad_f32 [[VX:v[0-9]]], [[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_y_add_x_negone(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -421,7 +438,10 @@ define void @test_f32_mul_y_add_x_negone(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_sub_one_x_y: -; SI: v_mad_f32 [[VX:v[0-9]]], -[[VX]], [[VY:v[0-9]]], [[VY]] +; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_sub_one_x_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -434,7 +454,10 @@ define void @test_f32_mul_sub_one_x_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_sub_one_x: -; SI: v_mad_f32 [[VX:v[0-9]]], -[[VX]], [[VY:v[0-9]]], [[VY]] +; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_y_sub_one_x(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -447,7 +470,10 @@ define void @test_f32_mul_y_sub_one_x(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_sub_negone_x_y: -; SI: v_mad_f32 [[VX:v[0-9]]], -[[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_sub_negone_x_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -460,7 +486,10 @@ define void @test_f32_mul_sub_negone_x_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_sub_negone_x: -; SI: v_mad_f32 [[VX:v[0-9]]], -[[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_y_sub_negone_x(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -473,7 +502,10 @@ define void @test_f32_mul_y_sub_negone_x(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_sub_x_one_y: -; SI: v_mad_f32 [[VX:v[0-9]]], [[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_sub_x_one_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -486,7 +518,10 @@ define void @test_f32_mul_sub_x_one_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_sub_x_one: -; SI: v_mad_f32 [[VX:v[0-9]]], [[VX]], [[VY:v[0-9]]], -[[VY]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]] define void @test_f32_mul_y_sub_x_one(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -499,7 +534,10 @@ define void @test_f32_mul_y_sub_x_one(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_sub_x_negone_y: -; SI: v_mac_f32_e32 [[VY:v[0-9]]], [[VY]], [[VX:v[0-9]]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_sub_x_negone_y(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -512,7 +550,10 @@ define void @test_f32_mul_sub_x_negone_y(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f32_mul_y_sub_x_negone: -; SI: v_mac_f32_e32 [[VY:v[0-9]]], [[VY]], [[VX:v[0-9]]] +; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]] +; +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]] define void @test_f32_mul_y_sub_x_negone(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2) { @@ -529,8 +570,12 @@ define void @test_f32_mul_y_sub_x_negone(float addrspace(1)* %out, ; ; FUNC-LABEL: {{^}}test_f32_interp: -; SI: v_mad_f32 [[VR:v[0-9]]], -[[VT:v[0-9]]], [[VY:v[0-9]]], [[VY]] -; SI: v_mac_f32_e32 [[VR]], [[VT]], [[VX:v[0-9]]] +; SI-NOFMA: v_sub_f32_e32 [[VT1:v[0-9]]], 1.0, [[VT:v[0-9]]] +; SI-NOFMA: v_mul_f32_e32 [[VTY:v[0-9]]], [[VT1]], [[VY:v[0-9]]] +; SI-NOFMA: v_mac_f32_e32 [[VTY]], [[VT]], [[VX:v[0-9]]] +; +; SI-FMA: v_fma_f32 [[VR:v[0-9]]], -[[VT:v[0-9]]], [[VY:v[0-9]]], [[VY]] +; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VT]], [[VR]] define void @test_f32_interp(float addrspace(1)* %out, float addrspace(1)* %in1, float addrspace(1)* %in2, @@ -547,8 +592,12 @@ define void @test_f32_interp(float addrspace(1)* %out, } ; FUNC-LABEL: {{^}}test_f64_interp: -; SI: v_fma_f64 [[VR:v\[[0-9]+:[0-9]+\]]], -[[VT:v\[[0-9]+:[0-9]+\]]], [[VY:v\[[0-9]+:[0-9]+\]]], [[VY]] -; SI: v_fma_f64 [[VR:v\[[0-9]+:[0-9]+\]]], [[VX:v\[[0-9]+:[0-9]+\]]], [[VT]], [[VR]] +; SI-NOFMA: v_add_f64 [[VT1:v\[[0-9]+:[0-9]+\]]], -[[VT:v\[[0-9]+:[0-9]+\]]], 1.0 +; SI-NOFMA: v_mul_f64 [[VTY:v\[[0-9]+:[0-9]+\]]], [[VY:v\[[0-9]+:[0-9]+\]]], [[VT1]] +; SI-NOFMA: v_fma_f64 v{{\[[0-9]+:[0-9]+\]}}, [[VX:v\[[0-9]+:[0-9]+\]]], [[VT]], [[VTY]] +; +; SI-FMA: v_fma_f64 [[VR:v\[[0-9]+:[0-9]+\]]], -[[VT:v\[[0-9]+:[0-9]+\]]], [[VY:v\[[0-9]+:[0-9]+\]]], [[VY]] +; SI-FMA: v_fma_f64 v{{\[[0-9]+:[0-9]+\]}}, [[VX:v\[[0-9]+:[0-9]+\]]], [[VT]], [[VR]] define void @test_f64_interp(double addrspace(1)* %out, double addrspace(1)* %in1, double addrspace(1)* %in2, diff --git a/test/CodeGen/AMDGPU/fmax3.f64.ll b/test/CodeGen/AMDGPU/fmax3.f64.ll index 9bbfe1e95c5b..4dae566d8489 100644 --- a/test/CodeGen/AMDGPU/fmax3.f64.ll +++ b/test/CodeGen/AMDGPU/fmax3.f64.ll @@ -6,8 +6,8 @@ declare double @llvm.maxnum.f64(double, double) nounwind readnone ; SI-LABEL: {{^}}test_fmax3_f64: ; SI-DAG: buffer_load_dwordx2 [[REGA:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+:[0-9]+}}], 0{{$}} ; SI-DAG: buffer_load_dwordx2 [[REGB:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:8 -; SI-DAG: buffer_load_dwordx2 [[REGC:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:16 ; SI: v_max_f64 [[REGA]], [[REGA]], [[REGB]] +; SI: buffer_load_dwordx2 [[REGC:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:16 ; SI: v_max_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[REGA]], [[REGC]] ; SI: buffer_store_dwordx2 [[RESULT]], ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/fmaxnum.ll b/test/CodeGen/AMDGPU/fmaxnum.ll index a2b33a794d99..6a7d760495bc 100644 --- a/test/CodeGen/AMDGPU/fmaxnum.ll +++ b/test/CodeGen/AMDGPU/fmaxnum.ll @@ -234,7 +234,7 @@ define void @constant_fold_fmax_f32_n0_n0(float addrspace(1)* %out) nounwind { } ; FUNC-LABEL: @fmax_var_immediate_f32 -; SI: v_max_f32_e64 {{v[0-9]+}}, 2.0, {{s[0-9]+}} +; SI: v_max_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, 2.0 ; EG: MEM_RAT_CACHELESS STORE_RAW [[OUT:T[0-9]+\.[XYZW]]] ; EG-NOT: MAX_DX10 @@ -246,7 +246,7 @@ define void @fmax_var_immediate_f32(float addrspace(1)* %out, float %a) nounwind } ; FUNC-LABEL: @fmax_immediate_var_f32 -; SI: v_max_f32_e64 {{v[0-9]+}}, 2.0, {{s[0-9]+}} +; SI: v_max_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, 2.0 ; EG: MEM_RAT_CACHELESS STORE_RAW [[OUT:T[0-9]+\.[XYZW]]] ; EG: MAX_DX10 {{.*}}[[OUT]], {{KC0\[[0-9]\].[XYZW]}}, literal.{{[xy]}} diff --git a/test/CodeGen/AMDGPU/fminnum.ll b/test/CodeGen/AMDGPU/fminnum.ll index 04cb01260bc0..9081a1e89511 100644 --- a/test/CodeGen/AMDGPU/fminnum.ll +++ b/test/CodeGen/AMDGPU/fminnum.ll @@ -233,7 +233,7 @@ define void @constant_fold_fmin_f32_n0_n0(float addrspace(1)* %out) nounwind { } ; FUNC-LABEL: @fmin_var_immediate_f32 -; SI: v_min_f32_e64 {{v[0-9]+}}, 2.0, {{s[0-9]+}} +; SI: v_min_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, 2.0 ; EG: MEM_RAT_CACHELESS STORE_RAW [[OUT:T[0-9]+\.[XYZW]]] ; EG: MIN_DX10 {{.*}}[[OUT]], {{KC0\[[0-9]\].[XYZW]}}, literal.{{[xy]}} @@ -244,7 +244,7 @@ define void @fmin_var_immediate_f32(float addrspace(1)* %out, float %a) nounwind } ; FUNC-LABEL: @fmin_immediate_var_f32 -; SI: v_min_f32_e64 {{v[0-9]+}}, 2.0, {{s[0-9]+}} +; SI: v_min_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, 2.0 ; EG: MEM_RAT_CACHELESS STORE_RAW [[OUT:T[0-9]+\.[XYZW]]] ; EG: MIN_DX10 {{.*}}[[OUT]], {{KC0\[[0-9]\].[XYZW]}}, literal.{{[xy]}} diff --git a/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll index 867c5c252b6c..9df336c2c489 100644 --- a/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll +++ b/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll @@ -1,17 +1,27 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; XUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s ; Make sure (fmul (fadd x, x), c) -> (fmul x, (fmul 2.0, c)) doesn't ; make add an instruction if the fadd has more than one use. +declare half @llvm.fabs.f16(half) #1 declare float @llvm.fabs.f32(float) #1 -; GCN-LABEL: {{^}}multiple_fadd_use_test: -; GCN: v_max_legacy_f32_e64 [[A16:v[0-9]+]], -; GCN: v_add_f32_e32 [[A17:v[0-9]+]], [[A16]], [[A16]] -; GCN: v_mul_f32_e32 [[A18:v[0-9]+]], [[A17]], [[A17]] -; GCN: v_mad_f32 [[A20:v[0-9]+]], -[[A18]], [[A17]], 1.0 -; GCN: buffer_store_dword [[A20]] -define void @multiple_fadd_use_test(float addrspace(1)* %out, float %x, float %y, float %z) #0 { +; GCN-LABEL: {{^}}multiple_fadd_use_test_f32: +; SI: v_max_legacy_f32_e64 [[A16:v[0-9]+]], +; SI: v_add_f32_e32 [[A17:v[0-9]+]], [[A16]], [[A16]] +; SI: v_mul_f32_e32 [[A18:v[0-9]+]], [[A17]], [[A17]] +; SI: v_mad_f32 [[A20:v[0-9]+]], -[[A18]], [[A17]], 1.0 +; SI: buffer_store_dword [[A20]] + +; VI: v_add_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -1.0 +; VI: v_add_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -1.0 +; VI: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, |v{{[0-9]+}}| +; VI: v_cndmask_b32_e32 +; VI: v_add_f32_e32 +; VI: v_mul_f32_e32 +; VI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0 +define void @multiple_fadd_use_test_f32(float addrspace(1)* %out, float %x, float %y, float %z) #0 { %a11 = fadd fast float %y, -1.0 %a12 = call float @llvm.fabs.f32(float %a11) %a13 = fadd fast float %x, -1.0 @@ -26,13 +36,13 @@ define void @multiple_fadd_use_test(float addrspace(1)* %out, float %x, float %y ret void } -; GCN-LABEL: {{^}}multiple_use_fadd_fmac +; GCN-LABEL: {{^}}multiple_use_fadd_fmac_f32: ; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], [[X:s[0-9]+]], s{{[0-9]+}} -; GCN-DAG: v_mac_f32_e64 [[MAD:v[0-9]+]], 2.0, [[X]] +; GCN-DAG: v_mac_f32_e64 [[MAD:v[0-9]+]], [[X]], 2.0 ; GCN-DAG: buffer_store_dword [[MUL2]] ; GCN-DAG: buffer_store_dword [[MAD]] ; GCN: s_endpgm -define void @multiple_use_fadd_fmac(float addrspace(1)* %out, float %x, float %y) #0 { +define void @multiple_use_fadd_fmac_f32(float addrspace(1)* %out, float %x, float %y) #0 { %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 %mul2 = fmul fast float %x, 2.0 %mad = fadd fast float %mul2, %y @@ -41,13 +51,13 @@ define void @multiple_use_fadd_fmac(float addrspace(1)* %out, float %x, float %y ret void } -; GCN-LABEL: {{^}}multiple_use_fadd_fmad: +; GCN-LABEL: {{^}}multiple_use_fadd_fmad_f32: ; GCN-DAG: v_add_f32_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}| -; GCN-DAG: v_mad_f32 [[MAD:v[0-9]+]], 2.0, |[[X]]|, v{{[0-9]+}} +; GCN-DAG: v_mad_f32 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}} ; GCN-DAG: buffer_store_dword [[MUL2]] ; GCN-DAG: buffer_store_dword [[MAD]] ; GCN: s_endpgm -define void @multiple_use_fadd_fmad(float addrspace(1)* %out, float %x, float %y) #0 { +define void @multiple_use_fadd_fmad_f32(float addrspace(1)* %out, float %x, float %y) #0 { %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 %x.abs = call float @llvm.fabs.f32(float %x) %mul2 = fmul fast float %x.abs, 2.0 @@ -57,10 +67,10 @@ define void @multiple_use_fadd_fmad(float addrspace(1)* %out, float %x, float %y ret void } -; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad: -; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X:s[0-9]+]]|, v{{[0-9]+}} -; GCN: v_mad_f32 {{v[0-9]+}}, 2.0, |[[X]]|, v{{[0-9]+}} -define void @multiple_use_fadd_multi_fmad(float addrspace(1)* %out, float %x, float %y, float %z) #0 { +; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad_f32: +; GCN: v_mad_f32 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, v{{[0-9]+}} +; GCN: v_mad_f32 {{v[0-9]+}}, |[[X]]|, 2.0, v{{[0-9]+}} +define void @multiple_use_fadd_multi_fmad_f32(float addrspace(1)* %out, float %x, float %y, float %z) #0 { %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 %x.abs = call float @llvm.fabs.f32(float %x) %mul2 = fmul fast float %x.abs, 2.0 @@ -71,11 +81,11 @@ define void @multiple_use_fadd_multi_fmad(float addrspace(1)* %out, float %x, fl ret void } -; GCN-LABEL: {{^}}fmul_x2_xn2: -; GCN: v_mul_f32_e64 [[TMP0:v[0-9]+]], -4.0, [[X:s[0-9]+]] +; GCN-LABEL: {{^}}fmul_x2_xn2_f32: +; GCN: v_mul_f32_e64 [[TMP0:v[0-9]+]], [[X:s[0-9]+]], -4.0 ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]] ; GCN: buffer_store_dword [[RESULT]] -define void @fmul_x2_xn2(float addrspace(1)* %out, float %x, float %y) #0 { +define void @fmul_x2_xn2_f32(float addrspace(1)* %out, float %x, float %y) #0 { %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 %mul2 = fmul fast float %x, 2.0 %muln2 = fmul fast float %x, -2.0 @@ -84,12 +94,12 @@ define void @fmul_x2_xn2(float addrspace(1)* %out, float %x, float %y) #0 { ret void } -; GCN-LABEL: {{^}}fmul_x2_xn3: +; GCN-LABEL: {{^}}fmul_x2_xn3_f32: ; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0xc0c00000 ; GCN: v_mul_f32_e32 [[TMP0:v[0-9]+]], [[X:s[0-9]+]], [[K]] ; GCN: v_mul_f32_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]] ; GCN: buffer_store_dword [[RESULT]] -define void @fmul_x2_xn3(float addrspace(1)* %out, float %x, float %y) #0 { +define void @fmul_x2_xn3_f32(float addrspace(1)* %out, float %x, float %y) #0 { %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1 %mul2 = fmul fast float %x, 2.0 %muln2 = fmul fast float %x, -3.0 @@ -98,5 +108,114 @@ define void @fmul_x2_xn3(float addrspace(1)* %out, float %x, float %y) #0 { ret void } +; GCN-LABEL: {{^}}multiple_fadd_use_test_f16: +; VI: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -1.0 +; VI: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -1.0 +; VI: v_cmp_gt_f16_e64 vcc, |v{{[0-9]+}}|, |v{{[0-9]+}}| +; VI: v_cndmask_b32_e32 +; VI: v_add_f16_e32 +; VI: v_mul_f16_e32 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0 +define void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %z = bitcast i16 %z.arg to half + %a11 = fadd fast half %y, -1.0 + %a12 = call half @llvm.fabs.f16(half %a11) + %a13 = fadd fast half %x, -1.0 + %a14 = call half @llvm.fabs.f16(half %a13) + %a15 = fcmp ogt half %a12, %a14 + %a16 = select i1 %a15, half %a12, half %a14 + %a17 = fmul fast half %a16, 2.0 + %a18 = fmul fast half %a17, %a17 + %a19 = fmul fast half %a18, %a17 + %a20 = fsub fast half 1.0, %a19 + store half %a20, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}multiple_use_fadd_fmac_f16: +; GCN-DAG: v_add_f16_e64 [[MUL2:v[0-9]+]], [[X:s[0-9]+]], s{{[0-9]+}} +; GCN-DAG: v_mac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0 +; GCN-DAG: buffer_store_short [[MUL2]] +; GCN-DAG: buffer_store_short [[MAD]] +; GCN: s_endpgm +define void @multiple_use_fadd_fmac_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1 + %mul2 = fmul fast half %x, 2.0 + %mad = fadd fast half %mul2, %y + store volatile half %mul2, half addrspace(1)* %out + store volatile half %mad, half addrspace(1)* %out.gep.1 + ret void +} + +; GCN-LABEL: {{^}}multiple_use_fadd_fmad_f16: +; GCN-DAG: v_add_f16_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |s{{[0-9]+}}| +; GCN-DAG: v_mad_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}} +; GCN-DAG: buffer_store_short [[MUL2]] +; GCN-DAG: buffer_store_short [[MAD]] +; GCN: s_endpgm +define void @multiple_use_fadd_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1 + %x.abs = call half @llvm.fabs.f16(half %x) + %mul2 = fmul fast half %x.abs, 2.0 + %mad = fadd fast half %mul2, %y + store volatile half %mul2, half addrspace(1)* %out + store volatile half %mad, half addrspace(1)* %out.gep.1 + ret void +} + +; GCN-LABEL: {{^}}multiple_use_fadd_multi_fmad_f16: +; GCN: v_mad_f16 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, v{{[0-9]+}} +; GCN: v_mad_f16 {{v[0-9]+}}, |[[X]]|, 2.0, v{{[0-9]+}} +define void @multiple_use_fadd_multi_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %z = bitcast i16 %z.arg to half + %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1 + %x.abs = call half @llvm.fabs.f16(half %x) + %mul2 = fmul fast half %x.abs, 2.0 + %mad0 = fadd fast half %mul2, %y + %mad1 = fadd fast half %mul2, %z + store volatile half %mad0, half addrspace(1)* %out + store volatile half %mad1, half addrspace(1)* %out.gep.1 + ret void +} + +; GCN-LABEL: {{^}}fmul_x2_xn2_f16: +; GCN: v_mul_f16_e64 [[TMP0:v[0-9]+]], [[X:s[0-9]+]], -4.0 +; GCN: v_mul_f16_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]] +; GCN: buffer_store_short [[RESULT]] +define void @fmul_x2_xn2_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1 + %mul2 = fmul fast half %x, 2.0 + %muln2 = fmul fast half %x, -2.0 + %mul = fmul fast half %mul2, %muln2 + store volatile half %mul, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fmul_x2_xn3_f16: +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0xc600 +; GCN: v_mul_f16_e32 [[TMP0:v[0-9]+]], [[X:s[0-9]+]], [[K]] +; GCN: v_mul_f16_e32 [[RESULT:v[0-9]+]], [[X]], [[TMP0]] +; GCN: buffer_store_short [[RESULT]] +define void @fmul_x2_xn3_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 { + %x = bitcast i16 %x.arg to half + %y = bitcast i16 %y.arg to half + %out.gep.1 = getelementptr half, half addrspace(1)* %out, i32 1 + %mul2 = fmul fast half %x, 2.0 + %muln2 = fmul fast half %x, -3.0 + %mul = fmul fast half %mul2, %muln2 + store volatile half %mul, half addrspace(1)* %out + ret void +} + attributes #0 = { nounwind "unsafe-fp-math"="true" } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fmul.f16.ll b/test/CodeGen/AMDGPU/fmul.f16.ll new file mode 100644 index 000000000000..9ce4d7684fe5 --- /dev/null +++ b/test/CodeGen/AMDGPU/fmul.f16.ll @@ -0,0 +1,150 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fmul_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mul_f16_e32 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fmul_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fmul half %a.val, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmul_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mul_f16_e32 v[[R_F16:[0-9]+]], 0x4200, v[[B_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fmul_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = fmul half 3.0, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmul_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mul_f16_e32 v[[R_F16:[0-9]+]], 4.0, v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fmul_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fmul half %a.val, 4.0 + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmul_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_mul_f16_e32 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_mul_f16_e32 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fmul_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fmul <2 x half> %a.val, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmul_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_mul_f16_e32 v[[R_F16_0:[0-9]+]], 0x4200, v[[B_V2_F16]] +; VI: v_mul_f16_e32 v[[R_F16_1:[0-9]+]], 4.0, v[[B_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fmul_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fmul <2 x half> <half 3.0, half 4.0>, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmul_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_mul_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]] +; VI: v_mul_f16_e32 v[[R_F16_1:[0-9]+]], 0x4200, v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fmul_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fmul <2 x half> %a.val, <half 4.0, half 3.0> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fmuladd.f16.ll b/test/CodeGen/AMDGPU/fmuladd.f16.ll new file mode 100644 index 000000000000..500b00bdcf87 --- /dev/null +++ b/test/CodeGen/AMDGPU/fmuladd.f16.ll @@ -0,0 +1,467 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-FLUSH,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-FLUSH,VI %s + +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI-DENORM-STRICT,VI-DENORM,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI-DENORM-CONTRACT,VI-DENORM,VI %s + +declare i32 @llvm.amdgcn.workitem.id.x() #1 +declare half @llvm.fmuladd.f16(half, half, half) #1 +declare half @llvm.fabs.f16(half) #1 + +; GCN-LABEL: {{^}}fmuladd_f16: +; VI-FLUSH: v_mac_f16_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} + +; VI-DENORM: v_fma_f16 {{v[0-9]+, v[0-9]+, v[0-9]+}} +define void @fmuladd_f16(half addrspace(1)* %out, half addrspace(1)* %in1, + half addrspace(1)* %in2, half addrspace(1)* %in3) #0 { + %r0 = load half, half addrspace(1)* %in1 + %r1 = load half, half addrspace(1)* %in2 + %r2 = load half, half addrspace(1)* %in3 + %r3 = tail call half @llvm.fmuladd.f16(half %r0, half %r1, half %r2) + store half %r3, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_a_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1, half %r2) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_a_2.0_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_a_2.0_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r3 = tail call half @llvm.fmuladd.f16(half %r1, half 2.0, half %r2) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f16: +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; VI-DENORM-STRICT: v_add_f16_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_a_a_b_f16(half addrspace(1)* %out, + half addrspace(1)* %in1, + half addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r0 = load volatile half, half addrspace(1)* %gep.0 + %r1 = load volatile half, half addrspace(1)* %gep.1 + + %add.0 = fadd half %r0, %r0 + %add.1 = fadd half %add.0, %r1 + store half %add.1, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_b_a_a_f16: +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; VI-DENORM-STRICT: v_add_f16_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_b_a_a_f16(half addrspace(1)* %out, + half addrspace(1)* %in1, + half addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r0 = load volatile half, half addrspace(1)* %gep.0 + %r1 = load volatile half, half addrspace(1)* %gep.1 + + %add.0 = fadd half %r0, %r0 + %add.1 = fadd half %r1, %add.0 + store half %add.1, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_neg_2.0_a_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], -2.0, [[R1]] +; VI-DENORM: v_fma_f16 [[R2:v[0-9]+]], [[R1]], -2.0, [[R2]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] +define void @fmuladd_neg_2.0_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r3 = tail call half @llvm.fmuladd.f16(half -2.0, half %r1, half %r2) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_neg_2.0_neg_a_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], 2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], -[[R1]], -2.0, [[R2]] +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_neg_2.0_neg_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r1.fneg = fsub half -0.000000e+00, %r1 + + %r3 = tail call half @llvm.fmuladd.f16(half -2.0, half %r1.fneg, half %r2) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_neg_a_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], -2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], -[[R1]], 2.0, [[R2]] +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_neg_a_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r1.fneg = fsub half -0.000000e+00, %r1 + + %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1.fneg, half %r2) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_a_neg_b_f16 +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_a_neg_b_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %r2.fneg = fsub half -0.000000e+00, %r2 + + %r3 = tail call half @llvm.fmuladd.f16(half 2.0, half %r1, half %r2.fneg) + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}mad_sub_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] + +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]] + +; VI-DENORM-STRICT: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %mul = fmul half %a, %b + %sub = fsub half %mul, %c + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_inv_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]] + +; VI-DENORM-STRICT: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_inv_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %mul = fmul half %a, %b + %sub = fsub half %c, %mul + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_fabs_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]| + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]| + +; VI-DENORM-STRICT: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; VI-DENORM-STRICT: v_sub_f16_e64 [[RESULT:v[0-9]+]], [[TMP]], |[[REGC]]| + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_fabs_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %c.abs = call half @llvm.fabs.f16(half %c) #0 + %mul = fmul half %a, %b + %sub = fsub half %mul, %c.abs + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_fabs_inv_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] + +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]| + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]| + +; VI-DENORM-STRICT: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; VI-DENORM-STRICT: v_sub_f16_e64 [[RESULT:v[0-9]+]], |[[REGC]]|, [[TMP]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_fabs_inv_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %c.abs = call half @llvm.fabs.f16(half %c) #0 + %mul = fmul half %a, %b + %sub = fsub half %c.abs, %mul + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}neg_neg_mad_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] + +; VI-FLUSH: v_mac_f16_e32 [[REGC]], [[REGB]], [[REGA]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[REGC]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], [[REGC]] + +; VI-DENORM-STRICT: v_mul_f16_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; VI-DENORM-STRICT: v_add_f16_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @neg_neg_mad_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %nega = fsub half -0.000000e+00, %a + %negb = fsub half -0.000000e+00, %b + %mul = fmul half %nega, %negb + %sub = fadd half %mul, %c + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}mad_fabs_sub_f16: +; GCN: {{buffer|flat}}_load_ushort [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_ushort [[REGC:v[0-9]+]] + +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]] + +; VI-DENORM-STRICT: v_mul_f16_e64 [[TMP:v[0-9]+]], [[REGA]], |[[REGB]]| +; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_fabs_sub_f16(half addrspace(1)* noalias nocapture %out, half addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr half, half addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr half, half addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr half, half addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr half, half addrspace(1)* %out, i64 %tid.ext + %a = load volatile half, half addrspace(1)* %gep0, align 2 + %b = load volatile half, half addrspace(1)* %gep1, align 2 + %c = load volatile half, half addrspace(1)* %gep2, align 2 + %b.abs = call half @llvm.fabs.f16(half %b) #0 + %mul = fmul half %a, %b.abs + %sub = fsub half %mul, %c + store half %sub, half addrspace(1)* %outgep, align 2 + ret void +} + +; GCN-LABEL: {{^}}fsub_c_fadd_a_a_f16: +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], +; VI-FLUSH: v_mac_f16_e32 [[R2]], -2.0, [[R1]] +; VI-FLUSH: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[RESULT:v[0-9]+]], [[R1]], -2.0, [[R2]] + +; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; VI-DENORM: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fsub_c_fadd_a_a_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %add = fadd half %r1, %r1 + %r3 = fsub half %r2, %add + + store half %r3, half addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fsub_fadd_a_a_c_f16: +; GCN: {{buffer|flat}}_load_ushort [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_ushort [[R2:v[0-9]+]], + +; VI-FLUSH: v_mad_f16 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] + +; VI-DENORM-CONTRACT: v_fma_f16 [[R2]], [[R1]], 2.0, -[[R2]] + +; VI-DENORM-STRICT: v_add_f16_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; VI-DENORM-STRICT: v_subrev_f16_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fsub_fadd_a_a_c_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr half, half addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr half, half addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr half, half addrspace(1)* %out, i32 %tid + + %r1 = load volatile half, half addrspace(1)* %gep.0 + %r2 = load volatile half, half addrspace(1)* %gep.1 + + %add = fadd half %r1, %r1 + %r3 = fsub half %add, %r2 + + store half %r3, half addrspace(1)* %gep.out + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fmuladd.f32.ll b/test/CodeGen/AMDGPU/fmuladd.f32.ll new file mode 100644 index 000000000000..e4b1053ff25c --- /dev/null +++ b/test/CodeGen/AMDGPU/fmuladd.f32.ll @@ -0,0 +1,583 @@ +; RUN: llc -verify-machineinstrs -mcpu=tahiti -mattr=-fp32-denormals,+fast-fmaf -fp-contract=on < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH-STRICT,GCN-FLUSH,SI-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-STRICT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=tahiti -mattr=+fp32-denormals,+fast-fmaf -fp-contract=on < %s | FileCheck -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-STRICT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=verde -mattr=-fp32-denormals,-fast-fmaf -fp-contract=on < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH-STRICT,GCN-FLUSH,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-STRICT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=verde -mattr=+fp32-denormals,-fast-fmaf -fp-contract=on < %s | FileCheck -check-prefixes=GCN,GCN-DENORM-STRICT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-STRICT,SI %s + +; RUN: llc -verify-machineinstrs -mcpu=tahiti -mattr=-fp32-denormals,+fast-fmaf -fp-contract=fast < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH-CONTRACT,GCN-FLUSH,SI-FLUSH,GCN-FLUSH-FASTFMA,GCN-FLUSH-FASTFMA-CONTRACT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=tahiti -mattr=+fp32-denormals,+fast-fmaf -fp-contract=fast < %s | FileCheck -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-FASTFMA,GCN-DENORM-FASTFMA-CONTRACT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=verde -mattr=-fp32-denormals,-fast-fmaf -fp-contract=fast < %s | FileCheck -check-prefixes=GCN,GCN-FLUSH-CONTRACT,GCN-FLUSH,SI-FLUSH,GCN-FLUSH-SLOWFMA,GCN-FLUSH-SLOWFMA-CONTRACT,SI %s +; RUN: llc -verify-machineinstrs -mcpu=verde -mattr=+fp32-denormals,-fast-fmaf -fp-contract=fast < %s | FileCheck -check-prefixes=GCN,GCN-DENORM-CONTRACT,GCN-DENORM,SI-DENORM,GCN-DENORM-SLOWFMA,GCN-DENORM-SLOWFMA-CONTRACT,SI %s + +; Test all permutations of: fp32 denormals, fast fp contract, fp contract enabled for fmuladd, fmaf fast/slow. + +target triple = "amdgcn--" + + +declare i32 @llvm.amdgcn.workitem.id.x() #1 +declare float @llvm.fmuladd.f32(float, float, float) #1 +declare half @llvm.fmuladd.f16(half, half, half) #1 +declare float @llvm.fabs.f32(float) #1 + +; GCN-LABEL: {{^}}fmuladd_f32: +; GCN-FLUSH: v_mac_f32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} + +; GCN-DENORM-FASTFMA: v_fma_f32 {{v[0-9]+, v[0-9]+, v[0-9]+}} + +; GCN-DENORM-SLOWFMA: v_mul_f32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} +; GCN-DENORM-SLOWFMA: v_add_f32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} +define void @fmuladd_f32(float addrspace(1)* %out, float addrspace(1)* %in1, + float addrspace(1)* %in2, float addrspace(1)* %in3) #0 { + %r0 = load float, float addrspace(1)* %in1 + %r1 = load float, float addrspace(1)* %in2 + %r2 = load float, float addrspace(1)* %in3 + %r3 = tail call float @llvm.fmuladd.f32(float %r0, float %r1, float %r2) + store float %r3, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fmul_fadd_f32: +; GCN-FLUSH: v_mac_f32 + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 + +; GCN-DENORM-STRICT: v_mul_f32_e32 +; GCN-DENORM-STRICT: v_add_f32_e32 +define void @fmul_fadd_f32(float addrspace(1)* %out, float addrspace(1)* %in1, + float addrspace(1)* %in2, float addrspace(1)* %in3) #0 { + %r0 = load volatile float, float addrspace(1)* %in1 + %r1 = load volatile float, float addrspace(1)* %in2 + %r2 = load volatile float, float addrspace(1)* %in3 + %mul = fmul float %r0, %r1 + %add = fadd float %mul, %r2 + store float %add, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_a_b_f32 +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], 2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI-DENORM buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1, float %r2) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_a_2.0_b_f32 +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], 2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_a_2.0_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r3 = tail call float @llvm.fmuladd.f32(float %r1, float 2.0, float %r2) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], 2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; GCN-DENORM-STRICT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-STRICT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_a_a_b_f32(float addrspace(1)* %out, + float addrspace(1)* %in1, + float addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r0 = load volatile float, float addrspace(1)* %gep.0 + %r1 = load volatile float, float addrspace(1)* %gep.1 + + %add.0 = fadd float %r0, %r0 + %add.1 = fadd float %add.0, %r1 + store float %add.1, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_b_a_a_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], 2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, [[R2]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; GCN-DENORM-STRICT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-STRICT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_b_a_a_f32(float addrspace(1)* %out, + float addrspace(1)* %in1, + float addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r0 = load volatile float, float addrspace(1)* %gep.0 + %r1 = load volatile float, float addrspace(1)* %gep.1 + + %add.0 = fadd float %r0, %r0 + %add.1 = fadd float %r1, %add.0 + store float %add.1, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_neg_2.0_a_b_f32 +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], +; GCN-FLUSH: v_mac_f32_e32 [[R2]], -2.0, [[R1]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], -2.0, [[R2]] + +; GCN-DENORM-SLOWFMA: v_mul_f32_e32 [[TMP:v[0-9]+]], -2.0, [[R1]] +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_neg_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r3 = tail call float @llvm.fmuladd.f32(float -2.0, float %r1, float %r2) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_neg_2.0_neg_a_b_f32 +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], 2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], -[[R1]], -2.0, [[R2]] + +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r1.fneg = fsub float -0.000000e+00, %r1 + + %r3 = tail call float @llvm.fmuladd.f32(float -2.0, float %r1.fneg, float %r2) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_neg_a_b_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], + +; GCN-FLUSH: v_mac_f32_e32 [[R2]], -2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], -[[R1]], 2.0, [[R2]] + +; GCN-DENORM-SLOWFMA: v_mul_f32_e32 [[TMP:v[0-9]+]], -2.0, [[R1]] +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r1.fneg = fsub float -0.000000e+00, %r1 + + %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1.fneg, float %r2) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fmuladd_2.0_a_neg_b_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] +; SI-FLUSH: buffer_store_dword [[RESULT]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] + +; GCN-DENORM-FASTFMA: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] + +; GCN-DENORM-SLOWFMA: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fmuladd_2.0_a_neg_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %r2.fneg = fsub float -0.000000e+00, %r2 + + %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1, float %r2.fneg) + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}mad_sub_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; GCN-DENORM-STRICT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %mul = fmul float %a, %b + %sub = fsub float %mul, %c + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_inv_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] + +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] + +; GCN-DENORM-STRICT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %mul = fmul float %a, %b + %sub = fsub float %c, %mul + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_fabs_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]| + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]| + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_sub_f32_e64 [[RESULT:v[0-9]+]], [[TMP]], |[[REGC]]| + +; GCN-DENORM-STRICT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-STRICT: v_sub_f32_e64 [[RESULT:v[0-9]+]], [[TMP]], |[[REGC]]| + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_fabs_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %c.abs = call float @llvm.fabs.f32(float %c) #0 + %mul = fmul float %a, %b + %sub = fsub float %mul, %c.abs + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}mad_sub_fabs_inv_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]| + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]| + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_sub_f32_e64 [[RESULT:v[0-9]+]], |[[REGC]]|, [[TMP]] + +; GCN-DENORM-STRICT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-STRICT: v_sub_f32_e64 [[RESULT:v[0-9]+]], |[[REGC]]|, [[TMP]] + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_sub_fabs_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %c.abs = call float @llvm.fabs.f32(float %c) #0 + %mul = fmul float %a, %b + %sub = fsub float %c.abs, %mul + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}neg_neg_mad_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] + +; GCN-FLUSH: v_mac_f32_e32 [[REGC]], [[REGB]], [[REGA]] +; SI-FLUSH: buffer_store_dword [[REGC]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[REGC]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], [[REGC]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] + +; GCN-DENORM-STRICT: v_mul_f32_e32 [[TMP:v[0-9]+]], [[REGB]], [[REGA]] +; GCN-DENORM-STRICT: v_add_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[REGC]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %nega = fsub float -0.000000e+00, %a + %negb = fsub float -0.000000e+00, %b + %mul = fmul float %nega, %negb + %sub = fadd float %mul, %c + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}mad_fabs_sub_f32: +; GCN: {{buffer|flat}}_load_dword [[REGA:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGB:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[REGC:v[0-9]+]] +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_mul_f32_e64 [[TMP:v[0-9]+]], [[REGA]], |[[REGB]]| +; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; GCN-DENORM-STRICT: v_mul_f32_e64 [[TMP:v[0-9]+]], [[REGA]], |[[REGB]]| +; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[REGC]], [[TMP]] + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @mad_fabs_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #0 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext + %a = load volatile float, float addrspace(1)* %gep0, align 4 + %b = load volatile float, float addrspace(1)* %gep1, align 4 + %c = load volatile float, float addrspace(1)* %gep2, align 4 + %b.abs = call float @llvm.fabs.f32(float %b) #0 + %mul = fmul float %a, %b.abs + %sub = fsub float %mul, %c + store float %sub, float addrspace(1)* %outgep, align 4 + ret void +} + +; GCN-LABEL: {{^}}fsub_c_fadd_a_a_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], +; GCN-FLUSH: v_mac_f32_e32 [[R2]], -2.0, [[R1]] +; SI-FLUSH: buffer_store_dword [[R2]] +; VI-FLUSH: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[R2]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], -2.0, [[R2]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; GCN-DENORM-STRICT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[TMP]], [[R2]] + +; SI-DENORM: buffer_store_dword [[RESULT]] +; VI-DENORM: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fsub_c_fadd_a_a_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %add = fadd float %r1, %r1 + %r3 = fsub float %r2, %add + + store float %r3, float addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fsub_fadd_a_a_c_f32: +; GCN: {{buffer|flat}}_load_dword [[R1:v[0-9]+]], +; GCN: {{buffer|flat}}_load_dword [[R2:v[0-9]+]], +; GCN-FLUSH: v_mad_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] + +; GCN-DENORM-FASTFMA-CONTRACT: v_fma_f32 [[RESULT:v[0-9]+]], [[R1]], 2.0, -[[R2]] + +; GCN-DENORM-SLOWFMA-CONTRACT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-SLOWFMA-CONTRACT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; GCN-DENORM-STRICT: v_add_f32_e32 [[TMP:v[0-9]+]], [[R1]], [[R1]] +; GCN-DENORM-STRICT: v_subrev_f32_e32 [[RESULT:v[0-9]+]], [[R2]], [[TMP]] + +; SI: buffer_store_dword [[RESULT]] +; VI: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fsub_fadd_a_a_c_f32(float addrspace(1)* %out, float addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid + + %r1 = load volatile float, float addrspace(1)* %gep.0 + %r2 = load volatile float, float addrspace(1)* %gep.1 + + %add = fadd float %r1, %r1 + %r3 = fsub float %add, %r2 + + store float %r3, float addrspace(1)* %gep.out + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fmuladd.f64.ll b/test/CodeGen/AMDGPU/fmuladd.f64.ll new file mode 100644 index 000000000000..0af44ef200ea --- /dev/null +++ b/test/CodeGen/AMDGPU/fmuladd.f64.ll @@ -0,0 +1,182 @@ +; RUN: llc -march=amdgcn -mcpu=tahiti -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICTSI %s +; RUN: llc -march=amdgcn -mcpu=verde -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,SI %s +; RUN: llc -march=amdgcn -mcpu=tahiti -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,SI %s +; RUN: llc -march=amdgcn -mcpu=verde -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -fp-contract=on -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-STRICT,VI %s +; RUN: llc -march=amdgcn -mcpu=tonga -fp-contract=fast -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-CONTRACT,VI %s + +; GCN-LABEL: {{^}}fmuladd_f64: +; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}} +define void @fmuladd_f64(double addrspace(1)* %out, double addrspace(1)* %in1, + double addrspace(1)* %in2, double addrspace(1)* %in3) #0 { + %r0 = load double, double addrspace(1)* %in1 + %r1 = load double, double addrspace(1)* %in2 + %r2 = load double, double addrspace(1)* %in3 + %r3 = tail call double @llvm.fmuladd.f64(double %r0, double %r1, double %r2) + store double %r3, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fmul_fadd_f64: +; GCN-CONTRACT: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}} + +; GCN-STRICT: v_mul_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}} +; GCN-STRICT: v_add_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}} +define void @fmul_fadd_f64(double addrspace(1)* %out, double addrspace(1)* %in1, + double addrspace(1)* %in2, double addrspace(1)* %in3) #0 { + %r0 = load double, double addrspace(1)* %in1 + %r1 = load double, double addrspace(1)* %in2 + %r2 = load double, double addrspace(1)* %in3 + %tmp = fmul double %r0, %r1 + %r3 = fadd double %tmp, %r2 + store double %r3, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f64: +; GCN: {{buffer|flat}}_load_dwordx2 [[R1:v\[[0-9]+:[0-9]+\]]], +; GCN: {{buffer|flat}}_load_dwordx2 [[R2:v\[[0-9]+:[0-9]+\]]], + +; GCN-STRICT: v_add_f64 [[TMP:v\[[0-9]+:[0-9]+\]]], [[R1]], [[R1]] +; GCN-STRICT: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[TMP]], [[R2]] + +; GCN-CONTRACT: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[R1]], 2.0, [[R2]] + +; SI: buffer_store_dwordx2 [[RESULT]] +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_a_a_b_f64(double addrspace(1)* %out, + double addrspace(1)* %in1, + double addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr double, double addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr double, double addrspace(1)* %out, i32 %tid + + %r0 = load volatile double, double addrspace(1)* %gep.0 + %r1 = load volatile double, double addrspace(1)* %gep.1 + + %add.0 = fadd double %r0, %r0 + %add.1 = fadd double %add.0, %r1 + store double %add.1, double addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_b_a_a_f64: +; GCN: {{buffer|flat}}_load_dwordx2 [[R1:v\[[0-9]+:[0-9]+\]]], +; GCN: {{buffer|flat}}_load_dwordx2 [[R2:v\[[0-9]+:[0-9]+\]]], + +; GCN-STRICT: v_add_f64 [[TMP:v\[[0-9]+:[0-9]+\]]], [[R1]], [[R1]] +; GCN-STRICT: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[R2]], [[TMP]] + +; GCN-CONTRACT: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[R1]], 2.0, [[R2]] + +; SI: buffer_store_dwordx2 [[RESULT]] +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] +define void @fadd_b_a_a_f64(double addrspace(1)* %out, + double addrspace(1)* %in1, + double addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr double, double addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr double, double addrspace(1)* %out, i32 %tid + + %r0 = load volatile double, double addrspace(1)* %gep.0 + %r1 = load volatile double, double addrspace(1)* %gep.1 + + %add.0 = fadd double %r0, %r0 + %add.1 = fadd double %r1, %add.0 + store double %add.1, double addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}mad_sub_f64: +; GCN-STRICT: v_mul_f64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} +; GCN-STRICT: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}} + +; GCN-CONTRACT: v_fma_f64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}} +define void @mad_sub_f64(double addrspace(1)* noalias nocapture %out, double addrspace(1)* noalias nocapture readonly %ptr) #1 { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tid.ext = sext i32 %tid to i64 + %gep0 = getelementptr double, double addrspace(1)* %ptr, i64 %tid.ext + %add1 = add i64 %tid.ext, 1 + %gep1 = getelementptr double, double addrspace(1)* %ptr, i64 %add1 + %add2 = add i64 %tid.ext, 2 + %gep2 = getelementptr double, double addrspace(1)* %ptr, i64 %add2 + %outgep = getelementptr double, double addrspace(1)* %out, i64 %tid.ext + %a = load volatile double, double addrspace(1)* %gep0, align 8 + %b = load volatile double, double addrspace(1)* %gep1, align 8 + %c = load volatile double, double addrspace(1)* %gep2, align 8 + %mul = fmul double %a, %b + %sub = fsub double %mul, %c + store double %sub, double addrspace(1)* %outgep, align 8 + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f64_fast_add0: +; GCN-STRICT: v_add_f64 +; GCN-STRICT: v_add_f64 + +; GCN-CONTRACT: v_fma_f64 +define void @fadd_a_a_b_f64_fast_add0(double addrspace(1)* %out, + double addrspace(1)* %in1, + double addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr double, double addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr double, double addrspace(1)* %out, i32 %tid + + %r0 = load volatile double, double addrspace(1)* %gep.0 + %r1 = load volatile double, double addrspace(1)* %gep.1 + + %add.0 = fadd fast double %r0, %r0 + %add.1 = fadd double %add.0, %r1 + store double %add.1, double addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f64_fast_add1: +; GCN-STRICT: v_add_f64 +; GCN-STRICT: v_add_f64 + +; GCN-CONTRACT: v_fma_f64 +define void @fadd_a_a_b_f64_fast_add1(double addrspace(1)* %out, + double addrspace(1)* %in1, + double addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr double, double addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr double, double addrspace(1)* %out, i32 %tid + + %r0 = load volatile double, double addrspace(1)* %gep.0 + %r1 = load volatile double, double addrspace(1)* %gep.1 + + %add.0 = fadd double %r0, %r0 + %add.1 = fadd fast double %add.0, %r1 + store double %add.1, double addrspace(1)* %gep.out + ret void +} + +; GCN-LABEL: {{^}}fadd_a_a_b_f64_fast: +; GCN: v_fma_f64 +define void @fadd_a_a_b_f64_fast(double addrspace(1)* %out, + double addrspace(1)* %in1, + double addrspace(1)* %in2) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep.0 = getelementptr double, double addrspace(1)* %out, i32 %tid + %gep.1 = getelementptr double, double addrspace(1)* %gep.0, i32 1 + %gep.out = getelementptr double, double addrspace(1)* %out, i32 %tid + + %r0 = load volatile double, double addrspace(1)* %gep.0 + %r1 = load volatile double, double addrspace(1)* %gep.1 + + %add.0 = fadd fast double %r0, %r0 + %add.1 = fadd fast double %add.0, %r1 + store double %add.1, double addrspace(1)* %gep.out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 +declare double @llvm.fmuladd.f64(double, double, double) #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fmuladd.ll b/test/CodeGen/AMDGPU/fmuladd.ll deleted file mode 100644 index c80374df4950..000000000000 --- a/test/CodeGen/AMDGPU/fmuladd.ll +++ /dev/null @@ -1,199 +0,0 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s - -declare float @llvm.fmuladd.f32(float, float, float) -declare double @llvm.fmuladd.f64(double, double, double) -declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone -declare float @llvm.fabs.f32(float) nounwind readnone - -; CHECK-LABEL: {{^}}fmuladd_f32: -; CHECK: v_mac_f32_e32 {{v[0-9]+, v[0-9]+, v[0-9]+}} - -define void @fmuladd_f32(float addrspace(1)* %out, float addrspace(1)* %in1, - float addrspace(1)* %in2, float addrspace(1)* %in3) { - %r0 = load float, float addrspace(1)* %in1 - %r1 = load float, float addrspace(1)* %in2 - %r2 = load float, float addrspace(1)* %in3 - %r3 = tail call float @llvm.fmuladd.f32(float %r0, float %r1, float %r2) - store float %r3, float addrspace(1)* %out - ret void -} - -; CHECK-LABEL: {{^}}fmuladd_f64: -; CHECK: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}} - -define void @fmuladd_f64(double addrspace(1)* %out, double addrspace(1)* %in1, - double addrspace(1)* %in2, double addrspace(1)* %in3) { - %r0 = load double, double addrspace(1)* %in1 - %r1 = load double, double addrspace(1)* %in2 - %r2 = load double, double addrspace(1)* %in3 - %r3 = tail call double @llvm.fmuladd.f64(double %r0, double %r1, double %r2) - store double %r3, double addrspace(1)* %out - ret void -} - -; CHECK-LABEL: {{^}}fmuladd_2.0_a_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fmuladd_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1, float %r2) - store float %r3, float addrspace(1)* %gep.out - ret void -} - -; CHECK-LABEL: {{^}}fmuladd_a_2.0_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fmuladd_a_2.0_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r3 = tail call float @llvm.fmuladd.f32(float %r1, float 2.0, float %r2) - store float %r3, float addrspace(1)* %gep.out - ret void -} - -; CHECK-LABEL: {{^}}fadd_a_a_b_f32: -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fadd_a_a_b_f32(float addrspace(1)* %out, - float addrspace(1)* %in1, - float addrspace(1)* %in2) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r0 = load volatile float, float addrspace(1)* %gep.0 - %r1 = load volatile float, float addrspace(1)* %gep.1 - - %add.0 = fadd float %r0, %r0 - %add.1 = fadd float %add.0, %r1 - store float %add.1, float addrspace(1)* %out - ret void -} - -; CHECK-LABEL: {{^}}fadd_b_a_a_f32: -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fadd_b_a_a_f32(float addrspace(1)* %out, - float addrspace(1)* %in1, - float addrspace(1)* %in2) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r0 = load volatile float, float addrspace(1)* %gep.0 - %r1 = load volatile float, float addrspace(1)* %gep.1 - - %add.0 = fadd float %r0, %r0 - %add.1 = fadd float %r1, %add.0 - store float %add.1, float addrspace(1)* %out - ret void -} - -; CHECK-LABEL: {{^}}fmuladd_neg_2.0_a_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], -2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fmuladd_neg_2.0_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r3 = tail call float @llvm.fmuladd.f32(float -2.0, float %r1, float %r2) - store float %r3, float addrspace(1)* %gep.out - ret void -} - - -; CHECK-LABEL: {{^}}fmuladd_neg_2.0_neg_a_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], 2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fmuladd_neg_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r1.fneg = fsub float -0.000000e+00, %r1 - - %r3 = tail call float @llvm.fmuladd.f32(float -2.0, float %r1.fneg, float %r2) - store float %r3, float addrspace(1)* %gep.out - ret void -} - - -; CHECK-LABEL: {{^}}fmuladd_2.0_neg_a_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mac_f32_e32 [[R2]], -2.0, [[R1]] -; CHECK: buffer_store_dword [[R2]] -define void @fmuladd_2.0_neg_a_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r1.fneg = fsub float -0.000000e+00, %r1 - - %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1.fneg, float %r2) - store float %r3, float addrspace(1)* %gep.out - ret void -} - - -; CHECK-LABEL: {{^}}fmuladd_2.0_a_neg_b_f32 -; CHECK-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; CHECK-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; CHECK: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]] -; CHECK: buffer_store_dword [[RESULT]] -define void @fmuladd_2.0_a_neg_b_f32(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %r2.fneg = fsub float -0.000000e+00, %r2 - - %r3 = tail call float @llvm.fmuladd.f32(float 2.0, float %r1, float %r2.fneg) - store float %r3, float addrspace(1)* %gep.out - ret void -} diff --git a/test/CodeGen/AMDGPU/fneg-fabs.f16.ll b/test/CodeGen/AMDGPU/fneg-fabs.f16.ll new file mode 100644 index 000000000000..d7d21311c1b9 --- /dev/null +++ b/test/CodeGen/AMDGPU/fneg-fabs.f16.ll @@ -0,0 +1,113 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=FUNC %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC %s + +; GCN-LABEL: {{^}}fneg_fabs_fadd_f16: +; CI: v_cvt_f32_f16_e32 +; CI: v_cvt_f32_f16_e32 +; CI: v_sub_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, |v{{[0-9]+}}| + +; VI-NOT: and +; VI: v_sub_f16_e64 {{v[0-9]+}}, {{v[0-9]+}}, |{{v[0-9]+}}| +define void @fneg_fabs_fadd_f16(half addrspace(1)* %out, half %x, half %y) { + %fabs = call half @llvm.fabs.f16(half %x) + %fsub = fsub half -0.000000e+00, %fabs + %fadd = fadd half %y, %fsub + store half %fadd, half addrspace(1)* %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}fneg_fabs_fmul_f16: +; CI: v_cvt_f32_f16_e32 +; CI: v_cvt_f32_f16_e32 +; CI: v_mul_f32_e64 {{v[0-9]+}}, {{v[0-9]+}}, -|{{v[0-9]+}}| +; CI: v_cvt_f16_f32_e32 + +; VI-NOT: and +; VI: v_mul_f16_e64 {{v[0-9]+}}, {{v[0-9]+}}, -|{{v[0-9]+}}| +; VI-NOT: and +define void @fneg_fabs_fmul_f16(half addrspace(1)* %out, half %x, half %y) { + %fabs = call half @llvm.fabs.f16(half %x) + %fsub = fsub half -0.000000e+00, %fabs + %fmul = fmul half %y, %fsub + store half %fmul, half addrspace(1)* %out, align 2 + ret void +} + +; DAGCombiner will transform: +; (fabs (f16 bitcast (i16 a))) => (f16 bitcast (and (i16 a), 0x7FFFFFFF)) +; unless isFabsFree returns true + +; GCN-LABEL: {{^}}fneg_fabs_free_f16: +; GCN: v_or_b32_e32 v{{[0-9]+}}, 0x8000, v{{[0-9]+}} +define void @fneg_fabs_free_f16(half addrspace(1)* %out, i16 %in) { + %bc = bitcast i16 %in to half + %fabs = call half @llvm.fabs.f16(half %bc) + %fsub = fsub half -0.000000e+00, %fabs + store half %fsub, half addrspace(1)* %out + ret void +} + +; FIXME: Should use or +; GCN-LABEL: {{^}}fneg_fabs_f16: +; CI: v_cvt_f32_f16_e32 v{{[0-9]+}}, +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| + +; VI: v_or_b32_e32 v{{[0-9]+}}, 0x8000, v{{[0-9]+}} +define void @fneg_fabs_f16(half addrspace(1)* %out, half %in) { + %fabs = call half @llvm.fabs.f16(half %in) + %fsub = fsub half -0.000000e+00, %fabs + store half %fsub, half addrspace(1)* %out, align 2 + ret void +} + +; GCN-LABEL: {{^}}v_fneg_fabs_f16: +; CI: v_cvt_f32_f16_e32 v{{[0-9]+}}, +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| + +; VI: v_or_b32_e32 v{{[0-9]+}}, 0x8000, v{{[0-9]+}} +define void @v_fneg_fabs_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %val = load half, half addrspace(1)* %in, align 2 + %fabs = call half @llvm.fabs.f16(half %val) + %fsub = fsub half -0.000000e+00, %fabs + store half %fsub, half addrspace(1)* %out, align 2 + ret void +} + +; FIXME: single bit op +; GCN-LABEL: {{^}}fneg_fabs_v2f16: +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| + +; VI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}} +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: flat_store_dword +define void @fneg_fabs_v2f16(<2 x half> addrspace(1)* %out, <2 x half> %in) { + %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %in) + %fsub = fsub <2 x half> <half -0.000000e+00, half -0.000000e+00>, %fabs + store <2 x half> %fsub, <2 x half> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}fneg_fabs_v4f16: +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| +; CI: v_cvt_f16_f32_e64 v{{[0-9]+}}, -|v{{[0-9]+}}| + +; VI: s_mov_b32 [[MASK:s[0-9]+]], 0x8000{{$}} +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: v_or_b32_e32 v{{[0-9]+}}, [[MASK]], +; VI: flat_store_dwordx2 +define void @fneg_fabs_v4f16(<4 x half> addrspace(1)* %out, <4 x half> %in) { + %fabs = call <4 x half> @llvm.fabs.v4f16(<4 x half> %in) + %fsub = fsub <4 x half> <half -0.000000e+00, half -0.000000e+00, half -0.000000e+00, half -0.000000e+00>, %fabs + store <4 x half> %fsub, <4 x half> addrspace(1)* %out + ret void +} + +declare half @llvm.fabs.f16(half) readnone +declare <2 x half> @llvm.fabs.v2f16(<2 x half>) readnone +declare <4 x half> @llvm.fabs.v4f16(<4 x half>) readnone diff --git a/test/CodeGen/AMDGPU/fneg-fabs.f64.ll b/test/CodeGen/AMDGPU/fneg-fabs.f64.ll index b03f318f4571..62c73c6b93d4 100644 --- a/test/CodeGen/AMDGPU/fneg-fabs.f64.ll +++ b/test/CodeGen/AMDGPU/fneg-fabs.f64.ll @@ -1,11 +1,11 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s ; FIXME: Check something here. Currently it seems fabs + fneg aren't ; into 2 modifiers, although theoretically that should work. ; GCN-LABEL: {{^}}fneg_fabs_fadd_f64: -; GCN: v_add_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}| +; GCN: v_add_f64 {{v\[[0-9]+:[0-9]+\]}}, -|v{{\[[0-9]+:[0-9]+\]}}|, {{s\[[0-9]+:[0-9]+\]}} define void @fneg_fabs_fadd_f64(double addrspace(1)* %out, double %x, double %y) { %fabs = call double @llvm.fabs.f64(double %x) %fsub = fsub double -0.000000e+00, %fabs @@ -25,7 +25,7 @@ define void @v_fneg_fabs_fadd_f64(double addrspace(1)* %out, double addrspace(1) } ; GCN-LABEL: {{^}}fneg_fabs_fmul_f64: -; GCN: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, -|{{v\[[0-9]+:[0-9]+\]}}| +; GCN: v_mul_f64 {{v\[[0-9]+:[0-9]+\]}}, -|{{v\[[0-9]+:[0-9]+\]}}|, {{s\[[0-9]+:[0-9]+\]}} define void @fneg_fabs_fmul_f64(double addrspace(1)* %out, double %x, double %y) { %fabs = call double @llvm.fabs.f64(double %x) %fsub = fsub double -0.000000e+00, %fabs diff --git a/test/CodeGen/AMDGPU/fneg-fabs.ll b/test/CodeGen/AMDGPU/fneg-fabs.ll index b99d2712ed75..9ee1171306c7 100644 --- a/test/CodeGen/AMDGPU/fneg-fabs.ll +++ b/test/CodeGen/AMDGPU/fneg-fabs.ll @@ -1,10 +1,10 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}fneg_fabs_fadd_f32: ; SI-NOT: and -; SI: v_sub_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, |{{v[0-9]+}}| +; SI: v_subrev_f32_e64 {{v[0-9]+}}, |{{v[0-9]+}}|, {{s[0-9]+}} define void @fneg_fabs_fadd_f32(float addrspace(1)* %out, float %x, float %y) { %fabs = call float @llvm.fabs.f32(float %x) %fsub = fsub float -0.000000e+00, %fabs @@ -15,7 +15,7 @@ define void @fneg_fabs_fadd_f32(float addrspace(1)* %out, float %x, float %y) { ; FUNC-LABEL: {{^}}fneg_fabs_fmul_f32: ; SI-NOT: and -; SI: v_mul_f32_e64 {{v[0-9]+}}, {{s[0-9]+}}, -|{{v[0-9]+}}| +; SI: v_mul_f32_e64 {{v[0-9]+}}, -|{{v[0-9]+}}|, {{s[0-9]+}} ; SI-NOT: and define void @fneg_fabs_fmul_f32(float addrspace(1)* %out, float %x, float %y) { %fabs = call float @llvm.fabs.f32(float %x) @@ -82,8 +82,10 @@ define void @v_fneg_fabs_f32(float addrspace(1)* %out, float addrspace(1)* %in) ; R600: |{{(PV|T[0-9])\.[XYZW]}}| ; R600: -PV -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} +; FIXME: In this case two uses of the constant should be folded +; SI: s_brev_b32 [[SIGNBITK:s[0-9]+]], 1{{$}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} define void @fneg_fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) { %fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in) %fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs @@ -92,10 +94,11 @@ define void @fneg_fabs_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %in) { } ; FUNC-LABEL: {{^}}fneg_fabs_v4f32: -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} -; SI: v_or_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} +; SI: s_brev_b32 [[SIGNBITK:s[0-9]+]], 1{{$}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} +; SI: v_or_b32_e32 v{{[0-9]+}}, [[SIGNBITK]], v{{[0-9]+}} define void @fneg_fabs_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %in) { %fabs = call <4 x float> @llvm.fabs.v4f32(<4 x float> %in) %fsub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %fabs diff --git a/test/CodeGen/AMDGPU/fneg.f16.ll b/test/CodeGen/AMDGPU/fneg.f16.ll new file mode 100644 index 000000000000..e3dfd9201a24 --- /dev/null +++ b/test/CodeGen/AMDGPU/fneg.f16.ll @@ -0,0 +1,61 @@ +; RUN: llc -march=amdgcn -mcpu=kaveri -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s + +; FIXME: Should be able to do scalar op +; FUNC-LABEL: {{^}}s_fneg_f16: + +define void @s_fneg_f16(half addrspace(1)* %out, half %in) { + %fneg = fsub half -0.000000e+00, %in + store half %fneg, half addrspace(1)* %out + ret void +} + +; FIXME: Should be able to use bit operations when illegal type as +; well. + +; FUNC-LABEL: {{^}}v_fneg_f16: +; GCN: flat_load_ushort [[VAL:v[0-9]+]], + +; CI: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[VAL]] +; CI: v_cvt_f16_f32_e64 [[CVT1:v[0-9]+]], -[[CVT0]] +; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[CVT1]] + +; VI: v_xor_b32_e32 [[XOR:v[0-9]+]], 0x8000, [[VAL]] +; VI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[XOR]] +define void @v_fneg_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %val = load half, half addrspace(1)* %in, align 2 + %fneg = fsub half -0.000000e+00, %val + store half %fneg, half addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}fneg_free_f16: +; GCN: flat_load_ushort [[NEG_VALUE:v[0-9]+]], + +; XCI: s_xor_b32 [[XOR:s[0-9]+]], [[NEG_VALUE]], 0x8000{{$}} +; CI: v_xor_b32_e32 [[XOR:v[0-9]+]], 0x8000, [[NEG_VALUE]] +; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[XOR]] +define void @fneg_free_f16(half addrspace(1)* %out, i16 %in) { + %bc = bitcast i16 %in to half + %fsub = fsub half -0.0, %bc + store half %fsub, half addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_fneg_fold_f16: +; GCN: flat_load_ushort [[NEG_VALUE:v[0-9]+]] + +; CI: v_cvt_f32_f16_e32 [[CVT0:v[0-9]+]], [[CVT0]] +; CI: v_mul_f32_e64 [[MUL:v[0-9]+]], -[[CVT0]], [[CVT0]] +; CI: v_cvt_f16_f32_e32 [[CVT1:v[0-9]+]], [[MUL]] +; CI: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, [[CVT1]] + +; VI-NOT: [[NEG_VALUE]] +; VI: v_mul_f16_e64 v{{[0-9]+}}, -[[NEG_VALUE]], [[NEG_VALUE]] +define void @v_fneg_fold_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %val = load half, half addrspace(1)* %in + %fsub = fsub half -0.0, %val + %fmul = fmul half %fsub, %val + store half %fmul, half addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/fneg.ll b/test/CodeGen/AMDGPU/fneg.ll index a0fd539863c6..941606ca0716 100644 --- a/test/CodeGen/AMDGPU/fneg.ll +++ b/test/CodeGen/AMDGPU/fneg.ll @@ -1,30 +1,30 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s -; FUNC-LABEL: {{^}}fneg_f32: +; FUNC-LABEL: {{^}}s_fneg_f32: ; R600: -PV ; GCN: v_xor_b32 -define void @fneg_f32(float addrspace(1)* %out, float %in) { +define void @s_fneg_f32(float addrspace(1)* %out, float %in) { %fneg = fsub float -0.000000e+00, %in store float %fneg, float addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}fneg_v2f32: +; FUNC-LABEL: {{^}}s_fneg_v2f32: ; R600: -PV ; R600: -PV ; GCN: v_xor_b32 ; GCN: v_xor_b32 -define void @fneg_v2f32(<2 x float> addrspace(1)* nocapture %out, <2 x float> %in) { +define void @s_fneg_v2f32(<2 x float> addrspace(1)* nocapture %out, <2 x float> %in) { %fneg = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %in store <2 x float> %fneg, <2 x float> addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}fneg_v4f32: +; FUNC-LABEL: {{^}}s_fneg_v4f32: ; R600: -PV ; R600: -T ; R600: -PV @@ -34,7 +34,7 @@ define void @fneg_v2f32(<2 x float> addrspace(1)* nocapture %out, <2 x float> %i ; GCN: v_xor_b32 ; GCN: v_xor_b32 ; GCN: v_xor_b32 -define void @fneg_v4f32(<4 x float> addrspace(1)* nocapture %out, <4 x float> %in) { +define void @s_fneg_v4f32(<4 x float> addrspace(1)* nocapture %out, <4 x float> %in) { %fneg = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %in store <4 x float> %fneg, <4 x float> addrspace(1)* %out ret void @@ -44,15 +44,31 @@ define void @fneg_v4f32(<4 x float> addrspace(1)* nocapture %out, <4 x float> %i ; (fneg (f32 bitcast (i32 a))) => (f32 bitcast (xor (i32 a), 0x80000000)) ; unless the target returns true for isNegFree() -; FUNC-LABEL: {{^}}fneg_free_f32: +; FUNC-LABEL: {{^}}fsub0_f32: + +; GCN: v_sub_f32_e64 v{{[0-9]}}, 0, s{{[0-9]+$}} + ; R600-NOT: XOR ; R600: -KC0[2].Z +define void @fsub0_f32(float addrspace(1)* %out, i32 %in) { + %bc = bitcast i32 %in to float + %fsub = fsub float 0.0, %bc + store float %fsub, float addrspace(1)* %out + ret void +} +; FUNC-LABEL: {{^}}fneg_free_f32: +; SI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb +; VI: s_load_dword [[NEG_VALUE:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; XXX: We could use v_add_f32_e64 with the negate bit here instead. -; GCN: v_sub_f32_e64 v{{[0-9]}}, 0, s{{[0-9]+$}} +; GCN: v_bfrev_b32_e32 [[SIGNBIT:v[0-9]+]], 1{{$}} +; GCN: v_xor_b32_e32 [[RES:v[0-9]+]], [[NEG_VALUE]], [[SIGNBIT]] +; GCN: buffer_store_dword [[RES]] + +; R600-NOT: XOR +; R600: -PV.W define void @fneg_free_f32(float addrspace(1)* %out, i32 %in) { %bc = bitcast i32 %in to float - %fsub = fsub float 0.0, %bc + %fsub = fsub float -0.0, %bc store float %fsub, float addrspace(1)* %out ret void } diff --git a/test/CodeGen/AMDGPU/fp_to_sint.ll b/test/CodeGen/AMDGPU/fp_to_sint.ll index 0cd0358bafd5..381c375151e8 100644 --- a/test/CodeGen/AMDGPU/fp_to_sint.ll +++ b/test/CodeGen/AMDGPU/fp_to_sint.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s --check-prefix=SI --check-prefix=FUNC +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s --check-prefix=SI --check-prefix=FUNC --check-prefix=GCN +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s --check-prefix=VI --check-prefix=FUNC --check-prefix=GCN ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck %s --check-prefix=EG --check-prefix=FUNC declare float @llvm.fabs.f32(float) #1 @@ -248,5 +248,14 @@ define void @fp_to_uint_fabs_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 { ret void } +; FUNC-LABEL: {{^}}fp_to_sint_f32_i16: +; GCN: v_cvt_i32_f32_e32 [[VAL:v[0-9]+]], s{{[0-9]+}} +; GCN: buffer_store_short [[VAL]] +define void @fp_to_sint_f32_i16(i16 addrspace(1)* %out, float %in) #0 { + %sint = fptosi float %in to i16 + store i16 %sint, i16 addrspace(1)* %out + ret void +} + attributes #0 = { nounwind } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fp_to_uint.ll b/test/CodeGen/AMDGPU/fp_to_uint.ll index 8a0f9fa2ac2b..d089c291f461 100644 --- a/test/CodeGen/AMDGPU/fp_to_uint.ll +++ b/test/CodeGen/AMDGPU/fp_to_uint.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -check-prefix=SI -check-prefix=FUNC -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s -check-prefix=SI -check-prefix=FUNC +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,FUNC,SI +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GCN,FUNC,VI ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck %s -check-prefix=EG -check-prefix=FUNC declare float @llvm.fabs.f32(float) #1 @@ -7,8 +7,8 @@ declare float @llvm.fabs.f32(float) #1 ; FUNC-LABEL: {{^}}fp_to_uint_f32_to_i32: ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW]}} -; SI: v_cvt_u32_f32_e32 -; SI: s_endpgm +; GCN: v_cvt_u32_f32_e32 +; GCN: s_endpgm define void @fp_to_uint_f32_to_i32 (i32 addrspace(1)* %out, float %in) { %conv = fptoui float %in to i32 store i32 %conv, i32 addrspace(1)* %out @@ -19,8 +19,8 @@ define void @fp_to_uint_f32_to_i32 (i32 addrspace(1)* %out, float %in) { ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW]}} ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -; SI: v_cvt_u32_f32_e32 -; SI: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 define void @fp_to_uint_v2f32_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x float> %in) { %result = fptoui <2 x float> %in to <2 x i32> store <2 x i32> %result, <2 x i32> addrspace(1)* %out @@ -32,10 +32,10 @@ define void @fp_to_uint_v2f32_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x float> ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW]}} ; EG: FLT_TO_UINT {{\** *}}T{{[0-9]+\.[XYZW], PV\.[XYZW]}} -; SI: v_cvt_u32_f32_e32 -; SI: v_cvt_u32_f32_e32 -; SI: v_cvt_u32_f32_e32 -; SI: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 define void @fp_to_uint_v4f32_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x float> addrspace(1)* %in) { %value = load <4 x float>, <4 x float> addrspace(1) * %in @@ -67,7 +67,7 @@ define void @fp_to_uint_v4f32_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x float> ; EG-DAG: CNDE_INT ; EG-DAG: CNDE_INT -; SI: s_endpgm +; GCN: s_endpgm define void @fp_to_uint_f32_to_i64(i64 addrspace(1)* %out, float %x) { %conv = fptoui float %x to i64 store i64 %conv, i64 addrspace(1)* %out @@ -118,7 +118,7 @@ define void @fp_to_uint_f32_to_i64(i64 addrspace(1)* %out, float %x) { ; EG-DAG: CNDE_INT ; EG-DAG: CNDE_INT -; SI: s_endpgm +; GCN: s_endpgm define void @fp_to_uint_v2f32_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x float> %x) { %conv = fptoui <2 x float> %x to <2 x i64> store <2 x i64> %conv, <2 x i64> addrspace(1)* %out @@ -211,7 +211,7 @@ define void @fp_to_uint_v2f32_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x float> ; EG-DAG: CNDE_INT ; EG-DAG: CNDE_INT -; SI: s_endpgm +; GCN: s_endpgm define void @fp_to_uint_v4f32_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x float> %x) { %conv = fptoui <4 x float> %x to <4 x i64> store <4 x i64> %conv, <4 x i64> addrspace(1)* %out @@ -220,7 +220,7 @@ define void @fp_to_uint_v4f32_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x float> ; FUNC-LABEL: {{^}}fp_to_uint_f32_to_i1: -; SI: v_cmp_eq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, 1.0, s{{[0-9]+}} +; GCN: v_cmp_eq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, 1.0, s{{[0-9]+}} ; EG: AND_INT ; EG: SETE_DX10 {{[*]?}} T{{[0-9]+}}.{{[XYZW]}}, KC0[2].Z, 1.0, @@ -231,7 +231,7 @@ define void @fp_to_uint_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 { } ; FUNC-LABEL: {{^}}fp_to_uint_fabs_f32_to_i1: -; SI: v_cmp_eq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, 1.0, |s{{[0-9]+}}| +; GCN: v_cmp_eq_f32_e64 s{{\[[0-9]+:[0-9]+\]}}, 1.0, |s{{[0-9]+}}| define void @fp_to_uint_fabs_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 { %in.fabs = call float @llvm.fabs.f32(float %in) %conv = fptoui float %in.fabs to i1 @@ -239,5 +239,18 @@ define void @fp_to_uint_fabs_f32_to_i1(i1 addrspace(1)* %out, float %in) #0 { ret void } +; FUNC-LABEL: {{^}}fp_to_uint_f32_to_i16: +; The reason different instructions are used on SI and VI is because for +; SI fp_to_uint is legalized by the type legalizer and for VI it is +; legalized by the dag legalizer and they legalize fp_to_uint differently. +; SI: v_cvt_u32_f32_e32 [[VAL:v[0-9]+]], s{{[0-9]+}} +; VI: v_cvt_i32_f32_e32 [[VAL:v[0-9]+]], s{{[0-9]+}} +; GCN: buffer_store_short [[VAL]] +define void @fp_to_uint_f32_to_i16(i16 addrspace(1)* %out, float %in) #0 { + %uint = fptoui float %in to i16 + store i16 %uint, i16 addrspace(1)* %out + ret void +} + attributes #0 = { nounwind } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/fpext.f16.ll b/test/CodeGen/AMDGPU/fpext.f16.ll new file mode 100644 index 000000000000..37796a999b18 --- /dev/null +++ b/test/CodeGen/AMDGPU/fpext.f16.ll @@ -0,0 +1,70 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fpext_f16_to_f32 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[R_F32:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_dword v[[R_F32]] +; GCN: s_endpgm +define void @fpext_f16_to_f32( + float addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fpext half %a.val to float + store float %r.val, float addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fpext_f16_to_f64 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_cvt_f64_f32_e32 v{{\[}}[[R_F64_0:[0-9]+]]:[[R_F64_1:[0-9]+]]{{\]}}, v[[A_F32]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_F64_0]]:[[R_F64_1]]{{\]}} +; GCN: s_endpgm +define void @fpext_f16_to_f64( + double addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fpext half %a.val to double + store double %r.val, double addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fpext_v2f16_to_v2f32 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; VI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[R_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[R_F32_1:[0-9]+]], v[[A_F16_1]] +; GCN: buffer_store_dwordx2 v{{\[}}[[R_F32_0]]:[[R_F32_1]]{{\]}} +; GCN: s_endpgm +define void @fpext_v2f16_to_v2f32( + <2 x float> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fpext <2 x half> %a.val to <2 x float> + store <2 x float> %r.val, <2 x float> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fpext_v2f16_to_v2f64 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_cvt_f64_f32_e32 v{{\[}}{{[0-9]+}}:[[R_F64_3:[0-9]+]]{{\]}}, v[[A_F32_1]] +; GCN: v_cvt_f64_f32_e32 v{{\[}}[[R_F64_0:[0-9]+]]:{{[0-9]+}}{{\]}}, v[[A_F32_0]] +; GCN: buffer_store_dwordx4 v{{\[}}[[R_F64_0]]:[[R_F64_3]]{{\]}} +; GCN: s_endpgm +define void @fpext_v2f16_to_v2f64( + <2 x double> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fpext <2 x half> %a.val to <2 x double> + store <2 x double> %r.val, <2 x double> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fptosi.f16.ll b/test/CodeGen/AMDGPU/fptosi.f16.ll new file mode 100644 index 000000000000..3c973e09d22b --- /dev/null +++ b/test/CodeGen/AMDGPU/fptosi.f16.ll @@ -0,0 +1,109 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fptosi_f16_to_i16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_cvt_i32_f32_e32 v[[R_I16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_I16]] +; GCN: s_endpgm +define void @fptosi_f16_to_i16( + i16 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptosi half %a.val to i16 + store i16 %r.val, i16 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptosi_f16_to_i32 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_cvt_i32_f32_e32 v[[R_I32:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fptosi_f16_to_i32( + i32 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptosi half %a.val to i32 + store i32 %r.val, i32 addrspace(1)* %r + ret void +} + +; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing +; test checks code generated for 'i64 = fp_to_sint f32'. + +; GCN-LABEL: {{^}}fptosi_f16_to_i64 +; GCN: buffer_load_ushort +; GCN: v_cvt_f32_f16_e32 +; GCN: s_endpgm +define void @fptosi_f16_to_i64( + i64 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptosi half %a.val to i64 + store i64 %r.val, i64 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptosi_v2f16_to_v2i16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_cvt_i32_f32_e32 v[[R_I16_0:[0-9]+]], v[[A_F32_0]] +; GCN: v_cvt_i32_f32_e32 v[[R_I16_1:[0-9]+]], v[[A_F32_1]] +; GCN: v_and_b32_e32 v[[R_I16_LO:[0-9]+]], 0xffff, v[[R_I16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_I16_HI:[0-9]+]], 16, v[[R_I16_1]] +; GCN: v_or_b32_e32 v[[R_V2_I16:[0-9]+]], v[[R_I16_HI]], v[[R_I16_LO]] +; GCN: buffer_store_dword v[[R_V2_I16]] +; GCN: s_endpgm +define void @fptosi_v2f16_to_v2i16( + <2 x i16> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptosi <2 x half> %a.val to <2 x i16> + store <2 x i16> %r.val, <2 x i16> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptosi_v2f16_to_v2i32 +; GCN: buffer_load_dword +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_i32_f32_e32 +; GCN: v_cvt_i32_f32_e32 +; GCN: buffer_store_dwordx2 +; GCN: s_endpgm +define void @fptosi_v2f16_to_v2i32( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptosi <2 x half> %a.val to <2 x i32> + store <2 x i32> %r.val, <2 x i32> addrspace(1)* %r + ret void +} + +; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing +; test checks code generated for 'i64 = fp_to_sint f32'. + +; GCN-LABEL: {{^}}fptosi_v2f16_to_v2i64 +; GCN: buffer_load_dword +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_f32_f16_e32 +; GCN: s_endpgm +define void @fptosi_v2f16_to_v2i64( + <2 x i64> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptosi <2 x half> %a.val to <2 x i64> + store <2 x i64> %r.val, <2 x i64> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fptoui.f16.ll b/test/CodeGen/AMDGPU/fptoui.f16.ll new file mode 100644 index 000000000000..a74d6d333dbe --- /dev/null +++ b/test/CodeGen/AMDGPU/fptoui.f16.ll @@ -0,0 +1,111 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fptoui_f16_to_i16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_u32_f32_e32 v[[R_I16:[0-9]+]], v[[A_F32]] +; VI: v_cvt_i32_f32_e32 v[[R_I16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_I16]] +; GCN: s_endpgm +define void @fptoui_f16_to_i16( + i16 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptoui half %a.val to i16 + store i16 %r.val, i16 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptoui_f16_to_i32 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_cvt_u32_f32_e32 v[[R_I32:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @fptoui_f16_to_i32( + i32 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptoui half %a.val to i32 + store i32 %r.val, i32 addrspace(1)* %r + ret void +} + +; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing +; test checks code generated for 'i64 = fp_to_uint f32'. + +; GCN-LABEL: {{^}}fptoui_f16_to_i64 +; GCN: buffer_load_ushort +; GCN: v_cvt_f32_f16_e32 +; GCN: s_endpgm +define void @fptoui_f16_to_i64( + i64 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fptoui half %a.val to i64 + store i64 %r.val, i64 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptoui_v2f16_to_v2i16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN-DAG: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; GCN-DAG: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_u32_f32_e32 v[[R_I16_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_u32_f32_e32 v[[R_I16_0:[0-9]+]], v[[A_F32_0]] +; VI: v_cvt_i32_f32_e32 v[[R_I16_0:[0-9]+]], v[[A_F32_0]] +; VI: v_cvt_i32_f32_e32 v[[R_I16_1:[0-9]+]], v[[A_F32_1]] +; GCN: v_lshlrev_b32_e32 v[[R_I16_HI:[0-9]+]], 16, v[[R_I16_1]] +; GCN: v_or_b32_e32 v[[R_V2_I16:[0-9]+]], v[[R_I16_HI]], v[[R_I16_0]] +; GCN: buffer_store_dword v[[R_V2_I16]] +; GCN: s_endpgm +define void @fptoui_v2f16_to_v2i16( + <2 x i16> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptoui <2 x half> %a.val to <2 x i16> + store <2 x i16> %r.val, <2 x i16> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptoui_v2f16_to_v2i32 +; GCN: buffer_load_dword +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: v_cvt_u32_f32_e32 +; GCN: buffer_store_dwordx2 +; GCN: s_endpgm +define void @fptoui_v2f16_to_v2i32( + <2 x i32> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptoui <2 x half> %a.val to <2 x i32> + store <2 x i32> %r.val, <2 x i32> addrspace(1)* %r + ret void +} + +; Need to make sure we promote f16 to f32 when converting f16 to i64. Existing +; test checks code generated for 'i64 = fp_to_uint f32'. + +; GCN-LABEL: {{^}}fptoui_v2f16_to_v2i64 +; GCN: buffer_load_dword +; GCN: v_cvt_f32_f16_e32 +; GCN: v_cvt_f32_f16_e32 +; GCN: s_endpgm +define void @fptoui_v2f16_to_v2i64( + <2 x i64> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fptoui <2 x half> %a.val to <2 x i64> + store <2 x i64> %r.val, <2 x i64> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fptrunc.f16.ll b/test/CodeGen/AMDGPU/fptrunc.f16.ll new file mode 100644 index 000000000000..a067960c5848 --- /dev/null +++ b/test/CodeGen/AMDGPU/fptrunc.f16.ll @@ -0,0 +1,72 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fptrunc_f32_to_f16 +; GCN: buffer_load_dword v[[A_F32:[0-9]+]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fptrunc_f32_to_f16( + half addrspace(1)* %r, + float addrspace(1)* %a) { +entry: + %a.val = load float, float addrspace(1)* %a + %r.val = fptrunc float %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptrunc_f64_to_f16 +; GCN: buffer_load_dwordx2 v{{\[}}[[A_F64_0:[0-9]+]]:[[A_F64_1:[0-9]+]]{{\]}} +; GCN: v_cvt_f32_f64_e32 v[[A_F32:[0-9]+]], v{{\[}}[[A_F64_0]]:[[A_F64_1]]{{\]}} +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fptrunc_f64_to_f16( + half addrspace(1)* %r, + double addrspace(1)* %a) { +entry: + %a.val = load double, double addrspace(1)* %a + %r.val = fptrunc double %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptrunc_v2f32_to_v2f16 +; GCN: buffer_load_dwordx2 v{{\[}}[[A_F32_0:[0-9]+]]:[[A_F32_1:[0-9]+]]{{\]}} +; GCN-DAG: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[A_F32_0]] +; GCN-DAG: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[A_F32_1]] +; GCN-DAG: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fptrunc_v2f32_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x float> addrspace(1)* %a) { +entry: + %a.val = load <2 x float>, <2 x float> addrspace(1)* %a + %r.val = fptrunc <2 x float> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fptrunc_v2f64_to_v2f16 +; GCN: buffer_load_dwordx4 v{{\[}}[[A_F64_0:[0-9]+]]:[[A_F64_3:[0-9]+]]{{\]}} +; GCN: v_cvt_f32_f64_e32 v[[A_F32_0:[0-9]+]], v{{\[}}[[A_F64_0]]:{{[0-9]+}}{{\]}} +; GCN: v_cvt_f32_f64_e32 v[[A_F32_1:[0-9]+]], v{{\[}}{{[0-9]+}}:[[A_F64_3]]{{\]}} +; GCN: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[A_F32_0]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[A_F32_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +define void @fptrunc_v2f64_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x double> addrspace(1)* %a) { +entry: + %a.val = load <2 x double>, <2 x double> addrspace(1)* %a + %r.val = fptrunc <2 x double> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fptrunc.ll b/test/CodeGen/AMDGPU/fptrunc.ll index 385e10e7baae..1fd125964bae 100644 --- a/test/CodeGen/AMDGPU/fptrunc.ll +++ b/test/CodeGen/AMDGPU/fptrunc.ll @@ -1,17 +1,29 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN-UNSAFE %s ; FUNC-LABEL: {{^}}fptrunc_f64_to_f32: -; SI: v_cvt_f32_f64_e32 {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} +; GCN: v_cvt_f32_f64_e32 {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} define void @fptrunc_f64_to_f32(float addrspace(1)* %out, double %in) { %result = fptrunc double %in to float store float %result, float addrspace(1)* %out ret void } +; FUNC-LABEL: {{^}}fptrunc_f64_to_f16: +; GCN-NOT: v_cvt +; GCN-UNSAFE: v_cvt_f32_f64_e32 [[F32:v[0-9]+]] +; GCN-UNSAFE: v_cvt_f16_f32_e32 v{{[0-9]+}}, [[F32]] +define void @fptrunc_f64_to_f16(i16 addrspace(1)* %out, double %in) { + %result = fptrunc double %in to half + %result_i16 = bitcast half %result to i16 + store i16 %result_i16, i16 addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}fptrunc_v2f64_to_v2f32: -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 define void @fptrunc_v2f64_to_v2f32(<2 x float> addrspace(1)* %out, <2 x double> %in) { %result = fptrunc <2 x double> %in to <2 x float> store <2 x float> %result, <2 x float> addrspace(1)* %out @@ -19,10 +31,10 @@ define void @fptrunc_v2f64_to_v2f32(<2 x float> addrspace(1)* %out, <2 x double> } ; FUNC-LABEL: {{^}}fptrunc_v4f64_to_v4f32: -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 define void @fptrunc_v4f64_to_v4f32(<4 x float> addrspace(1)* %out, <4 x double> %in) { %result = fptrunc <4 x double> %in to <4 x float> store <4 x float> %result, <4 x float> addrspace(1)* %out @@ -30,14 +42,14 @@ define void @fptrunc_v4f64_to_v4f32(<4 x float> addrspace(1)* %out, <4 x double> } ; FUNC-LABEL: {{^}}fptrunc_v8f64_to_v8f32: -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 -; SI: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 +; GCN: v_cvt_f32_f64_e32 define void @fptrunc_v8f64_to_v8f32(<8 x float> addrspace(1)* %out, <8 x double> %in) { %result = fptrunc <8 x double> %in to <8 x float> store <8 x float> %result, <8 x float> addrspace(1)* %out diff --git a/test/CodeGen/AMDGPU/fract.f64.ll b/test/CodeGen/AMDGPU/fract.f64.ll index 68b884363ec5..29b90a43ab68 100644 --- a/test/CodeGen/AMDGPU/fract.f64.ll +++ b/test/CodeGen/AMDGPU/fract.f64.ll @@ -13,9 +13,9 @@ declare double @llvm.floor.f64(double) #0 ; SI-DAG: v_mov_b32_e32 v[[UPLO:[0-9]+]], -1 ; SI-DAG: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff ; SI-DAG: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]] -; SI-DAG: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3 -; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]] -; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]] +; SI-DAG: v_cmp_class_f64_e64 vcc, v{{\[}}[[LO]]:[[HI]]], 3 +; SI: v_cndmask_b32_e32 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], vcc +; SI: v_cndmask_b32_e32 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], vcc ; SI: v_add_f64 [[SUB0:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]{{\]}}, -v{{\[}}[[RESLO]]:[[RESHI]]{{\]}} ; SI: v_add_f64 [[FRACT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]{{\]}}, -[[SUB0]] @@ -40,9 +40,9 @@ define void @fract_f64(double addrspace(1)* %out, double addrspace(1)* %src) #1 ; SI-DAG: v_mov_b32_e32 v[[UPLO:[0-9]+]], -1 ; SI-DAG: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff ; SI-DAG: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]] -; SI-DAG: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3 -; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]] -; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]] +; SI-DAG: v_cmp_class_f64_e64 vcc, v{{\[}}[[LO]]:[[HI]]], 3 +; SI: v_cndmask_b32_e32 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], vcc +; SI: v_cndmask_b32_e32 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], vcc ; SI: v_add_f64 [[SUB0:v\[[0-9]+:[0-9]+\]]], -v{{\[}}[[LO]]:[[HI]]{{\]}}, -v{{\[}}[[RESLO]]:[[RESHI]]{{\]}} ; SI: v_add_f64 [[FRACT:v\[[0-9]+:[0-9]+\]]], -v{{\[}}[[LO]]:[[HI]]{{\]}}, -[[SUB0]] @@ -68,9 +68,9 @@ define void @fract_f64_neg(double addrspace(1)* %out, double addrspace(1)* %src) ; SI-DAG: v_mov_b32_e32 v[[UPLO:[0-9]+]], -1 ; SI-DAG: v_mov_b32_e32 v[[UPHI:[0-9]+]], 0x3fefffff ; SI-DAG: v_min_f64 v{{\[}}[[MINLO:[0-9]+]]:[[MINHI:[0-9]+]]], v{{\[}}[[UPLO]]:[[UPHI]]], [[FRC]] -; SI-DAG: v_cmp_class_f64_e64 [[COND:s\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO]]:[[HI]]], 3 -; SI: v_cndmask_b32_e64 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], [[COND]] -; SI: v_cndmask_b32_e64 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], [[COND]] +; SI-DAG: v_cmp_class_f64_e64 vcc, v{{\[}}[[LO]]:[[HI]]], 3 +; SI: v_cndmask_b32_e32 v[[RESLO:[0-9]+]], v[[MINLO]], v[[LO]], vcc +; SI: v_cndmask_b32_e32 v[[RESHI:[0-9]+]], v[[MINHI]], v[[HI]], vcc ; SI: v_add_f64 [[SUB0:v\[[0-9]+:[0-9]+\]]], -|v{{\[}}[[LO]]:[[HI]]{{\]}}|, -v{{\[}}[[RESLO]]:[[RESHI]]{{\]}} ; SI: v_add_f64 [[FRACT:v\[[0-9]+:[0-9]+\]]], -|v{{\[}}[[LO]]:[[HI]]{{\]}}|, -[[SUB0]] diff --git a/test/CodeGen/AMDGPU/fsub.f16.ll b/test/CodeGen/AMDGPU/fsub.f16.ll new file mode 100644 index 000000000000..fb15edbaaffc --- /dev/null +++ b/test/CodeGen/AMDGPU/fsub.f16.ll @@ -0,0 +1,150 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}fsub_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_subrev_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_subrev_f16_e32 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fsub_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = fsub half %a.val, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fsub_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_subrev_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_sub_f16_e32 v[[R_F16:[0-9]+]], 1.0, v[[B_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fsub_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = fsub half 1.0, %b.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fsub_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0xc000{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_add_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_add_f16_e32 v[[R_F16:[0-9]+]], -2.0, v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fsub_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = fsub half %a.val, 2.0 + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fsub_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_subrev_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_subrev_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_subrev_f16_e32 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_subrev_f16_e32 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fsub_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fsub <2 x half> %a.val, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fsub_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], 0x4000{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_subrev_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_subrev_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_sub_f16_e32 v[[R_F16_0:[0-9]+]], 1.0, v[[B_V2_F16]] +; VI: v_sub_f16_e32 v[[R_F16_1:[0-9]+]], 2.0, v[[B_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fsub_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = fsub <2 x half> <half 1.0, half 2.0>, %b.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fsub_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], 0x4000{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], 0x3c00{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_subrev_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_subrev_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_add_f16_e32 v[[R_F16_0:[0-9]+]], -2.0, v[[A_V2_F16]] +; VI: v_add_f16_e32 v[[R_F16_1:[0-9]+]], -1.0, v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fsub_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = fsub <2 x half> %a.val, <half 2.0, half 1.0> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/fsub64.ll b/test/CodeGen/AMDGPU/fsub64.ll index f1b970a4f5fe..4c9c5ddd4c6e 100644 --- a/test/CodeGen/AMDGPU/fsub64.ll +++ b/test/CodeGen/AMDGPU/fsub64.ll @@ -39,7 +39,7 @@ define void @fsub_fabs_inv_f64(double addrspace(1)* %out, double addrspace(1)* % } ; SI-LABEL: {{^}}s_fsub_f64: -; SI: v_add_f64 {{v\[[0-9]+:[0-9]+\], s\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}} +; SI: v_add_f64 {{v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\], s\[[0-9]+:[0-9]+\]}} define void @s_fsub_f64(double addrspace(1)* %out, double %a, double %b) { %sub = fsub double %a, %b store double %sub, double addrspace(1)* %out diff --git a/test/CodeGen/AMDGPU/global-constant.ll b/test/CodeGen/AMDGPU/global-constant.ll index 0f2fc836a245..5a18d425d506 100644 --- a/test/CodeGen/AMDGPU/global-constant.ll +++ b/test/CodeGen/AMDGPU/global-constant.ll @@ -1,27 +1,54 @@ ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=NOHSA %s ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=HSA %s -@readonly = private unnamed_addr addrspace(2) constant [4 x float] [float 0.0, float 1.0, float 2.0, float 3.0] -@readonly2 = private unnamed_addr addrspace(2) constant [4 x float] [float 4.0, float 5.0, float 6.0, float 7.0] +@private1 = private unnamed_addr addrspace(2) constant [4 x float] [float 0.0, float 1.0, float 2.0, float 3.0] +@private2 = private unnamed_addr addrspace(2) constant [4 x float] [float 4.0, float 5.0, float 6.0, float 7.0] +@available_externally = available_externally addrspace(2) global [256 x i32] zeroinitializer -; GCN-LABEL: {{^}}main: +; GCN-LABEL: {{^}}private_test: ; GCN: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}} -; GCN-NEXT: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], readonly -; GCN: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], 0 + +; Non-HSA OSes use fixup into .text section. +; NOHSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1 +; NOHSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], 0 + +; HSA OSes use relocations. +; HSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], private1@rel32@lo+4 +; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], private1@rel32@hi+4 + ; GCN: s_getpc_b64 s{{\[}}[[PC1_LO:[0-9]+]]:[[PC1_HI:[0-9]+]]{{\]}} -; GCN-NEXT: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], readonly -; GCN: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], 0 -; NOHSA: .text -; HSA: .text -; GCN: readonly: -; GCN: readonly2: -define void @main(i32 %index, float addrspace(1)* %out) { - %ptr = getelementptr [4 x float], [4 x float] addrspace(2) * @readonly, i32 0, i32 %index + +; Non-HSA OSes use fixup into .text section. +; NOHSA: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2 +; NOHSA: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], 0 + +; HSA OSes use relocations. +; HSA: s_add_u32 s{{[0-9]+}}, s[[PC1_LO]], private2@rel32@lo+4 +; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC1_HI]], private2@rel32@hi+4 + +define void @private_test(i32 %index, float addrspace(1)* %out) { + %ptr = getelementptr [4 x float], [4 x float] addrspace(2) * @private1, i32 0, i32 %index %val = load float, float addrspace(2)* %ptr store float %val, float addrspace(1)* %out - %ptr2 = getelementptr [4 x float], [4 x float] addrspace(2) * @readonly2, i32 0, i32 %index + %ptr2 = getelementptr [4 x float], [4 x float] addrspace(2) * @private2, i32 0, i32 %index %val2 = load float, float addrspace(2)* %ptr2 store float %val2, float addrspace(1)* %out ret void } +; HSA-LABEL: {{^}}available_externally_test: +; HSA: s_getpc_b64 s{{\[}}[[PC0_LO:[0-9]+]]:[[PC0_HI:[0-9]+]]{{\]}} +; HSA: s_add_u32 s{{[0-9]+}}, s[[PC0_LO]], available_externally@gotpcrel32@lo+4 +; HSA: s_addc_u32 s{{[0-9]+}}, s[[PC0_HI]], available_externally@gotpcrel32@hi+4 +define void @available_externally_test(i32 addrspace(1)* %out) { + %ptr = getelementptr [256 x i32], [256 x i32] addrspace(2)* @available_externally, i32 0, i32 1 + %val = load i32, i32 addrspace(2)* %ptr + store i32 %val, i32 addrspace(1)* %out + ret void +} + +; NOHSA: .text +; HSA: .section .rodata + +; GCN: private1: +; GCN: private2: diff --git a/test/CodeGen/AMDGPU/global-extload-i16.ll b/test/CodeGen/AMDGPU/global-extload-i16.ll new file mode 100644 index 000000000000..4d9992997163 --- /dev/null +++ b/test/CodeGen/AMDGPU/global-extload-i16.ll @@ -0,0 +1,302 @@ +; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; XUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; FIXME: cypress is broken because the bigger testcases spill and it's not implemented + +; FUNC-LABEL: {{^}}zextload_global_i16_to_i32: +; SI: buffer_load_ushort +; SI: buffer_store_dword +; SI: s_endpgm +define void @zextload_global_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in) nounwind { + %a = load i16, i16 addrspace(1)* %in + %ext = zext i16 %a to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_i16_to_i32: +; SI: buffer_load_sshort +; SI: buffer_store_dword +; SI: s_endpgm +define void @sextload_global_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in) nounwind { + %a = load i16, i16 addrspace(1)* %in + %ext = sext i16 %a to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v1i16_to_v1i32: +; SI: buffer_load_ushort +; SI: s_endpgm +define void @zextload_global_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <1 x i16>, <1 x i16> addrspace(1)* %in + %ext = zext <1 x i16> %load to <1 x i32> + store <1 x i32> %ext, <1 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v1i16_to_v1i32: +; SI: buffer_load_sshort +; SI: s_endpgm +define void @sextload_global_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <1 x i16>, <1 x i16> addrspace(1)* %in + %ext = sext <1 x i16> %load to <1 x i32> + store <1 x i32> %ext, <1 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v2i16_to_v2i32: +; SI: s_endpgm +define void @zextload_global_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in + %ext = zext <2 x i16> %load to <2 x i32> + store <2 x i32> %ext, <2 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v2i16_to_v2i32: +; SI: s_endpgm +define void @sextload_global_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in + %ext = sext <2 x i16> %load to <2 x i32> + store <2 x i32> %ext, <2 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v4i16_to_v4i32: +; SI: s_endpgm +define void @zextload_global_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <4 x i16>, <4 x i16> addrspace(1)* %in + %ext = zext <4 x i16> %load to <4 x i32> + store <4 x i32> %ext, <4 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v4i16_to_v4i32: +; SI: s_endpgm +define void @sextload_global_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <4 x i16>, <4 x i16> addrspace(1)* %in + %ext = sext <4 x i16> %load to <4 x i32> + store <4 x i32> %ext, <4 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v8i16_to_v8i32: +; SI: s_endpgm +define void @zextload_global_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <8 x i16>, <8 x i16> addrspace(1)* %in + %ext = zext <8 x i16> %load to <8 x i32> + store <8 x i32> %ext, <8 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v8i16_to_v8i32: +; SI: s_endpgm +define void @sextload_global_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <8 x i16>, <8 x i16> addrspace(1)* %in + %ext = sext <8 x i16> %load to <8 x i32> + store <8 x i32> %ext, <8 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v16i16_to_v16i32: +; SI: s_endpgm +define void @zextload_global_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <16 x i16>, <16 x i16> addrspace(1)* %in + %ext = zext <16 x i16> %load to <16 x i32> + store <16 x i32> %ext, <16 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v16i16_to_v16i32: +; SI: s_endpgm +define void @sextload_global_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <16 x i16>, <16 x i16> addrspace(1)* %in + %ext = sext <16 x i16> %load to <16 x i32> + store <16 x i32> %ext, <16 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v32i16_to_v32i32: +; SI: s_endpgm +define void @zextload_global_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <32 x i16>, <32 x i16> addrspace(1)* %in + %ext = zext <32 x i16> %load to <32 x i32> + store <32 x i32> %ext, <32 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v32i16_to_v32i32: +; SI: s_endpgm +define void @sextload_global_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <32 x i16>, <32 x i16> addrspace(1)* %in + %ext = sext <32 x i16> %load to <32 x i32> + store <32 x i32> %ext, <32 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v64i16_to_v64i32: +; SI: s_endpgm +define void @zextload_global_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <64 x i16>, <64 x i16> addrspace(1)* %in + %ext = zext <64 x i16> %load to <64 x i32> + store <64 x i32> %ext, <64 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v64i16_to_v64i32: +; SI: s_endpgm +define void @sextload_global_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <64 x i16>, <64 x i16> addrspace(1)* %in + %ext = sext <64 x i16> %load to <64 x i32> + store <64 x i32> %ext, <64 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_i16_to_i64: +; SI-DAG: buffer_load_ushort v[[LO:[0-9]+]], +; SI-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}} +; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] +define void @zextload_global_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in) nounwind { + %a = load i16, i16 addrspace(1)* %in + %ext = zext i16 %a to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_i16_to_i64: +; VI: buffer_load_ushort [[LOAD:v[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0 +; VI: v_ashrrev_i32_e32 v{{[0-9]+}}, 31, [[LOAD]] +; VI: buffer_store_dwordx2 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 +define void @sextload_global_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in) nounwind { + %a = load i16, i16 addrspace(1)* %in + %ext = sext i16 %a to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v1i16_to_v1i64: +; SI: s_endpgm +define void @zextload_global_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <1 x i16>, <1 x i16> addrspace(1)* %in + %ext = zext <1 x i16> %load to <1 x i64> + store <1 x i64> %ext, <1 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v1i16_to_v1i64: +; SI: s_endpgm +define void @sextload_global_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <1 x i16>, <1 x i16> addrspace(1)* %in + %ext = sext <1 x i16> %load to <1 x i64> + store <1 x i64> %ext, <1 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v2i16_to_v2i64: +; SI: s_endpgm +define void @zextload_global_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in + %ext = zext <2 x i16> %load to <2 x i64> + store <2 x i64> %ext, <2 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v2i16_to_v2i64: +; SI: s_endpgm +define void @sextload_global_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in + %ext = sext <2 x i16> %load to <2 x i64> + store <2 x i64> %ext, <2 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v4i16_to_v4i64: +; SI: s_endpgm +define void @zextload_global_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <4 x i16>, <4 x i16> addrspace(1)* %in + %ext = zext <4 x i16> %load to <4 x i64> + store <4 x i64> %ext, <4 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v4i16_to_v4i64: +; SI: s_endpgm +define void @sextload_global_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <4 x i16>, <4 x i16> addrspace(1)* %in + %ext = sext <4 x i16> %load to <4 x i64> + store <4 x i64> %ext, <4 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v8i16_to_v8i64: +; SI: s_endpgm +define void @zextload_global_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <8 x i16>, <8 x i16> addrspace(1)* %in + %ext = zext <8 x i16> %load to <8 x i64> + store <8 x i64> %ext, <8 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v8i16_to_v8i64: +; SI: s_endpgm +define void @sextload_global_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <8 x i16>, <8 x i16> addrspace(1)* %in + %ext = sext <8 x i16> %load to <8 x i64> + store <8 x i64> %ext, <8 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v16i16_to_v16i64: +; SI: s_endpgm +define void @zextload_global_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <16 x i16>, <16 x i16> addrspace(1)* %in + %ext = zext <16 x i16> %load to <16 x i64> + store <16 x i64> %ext, <16 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v16i16_to_v16i64: +; SI: s_endpgm +define void @sextload_global_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <16 x i16>, <16 x i16> addrspace(1)* %in + %ext = sext <16 x i16> %load to <16 x i64> + store <16 x i64> %ext, <16 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v32i16_to_v32i64: +; SI: s_endpgm +define void @zextload_global_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <32 x i16>, <32 x i16> addrspace(1)* %in + %ext = zext <32 x i16> %load to <32 x i64> + store <32 x i64> %ext, <32 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v32i16_to_v32i64: +; SI: s_endpgm +define void @sextload_global_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <32 x i16>, <32 x i16> addrspace(1)* %in + %ext = sext <32 x i16> %load to <32 x i64> + store <32 x i64> %ext, <32 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}zextload_global_v64i16_to_v64i64: +; SI: s_endpgm +define void @zextload_global_v64i16_to_v64i64(<64 x i64> addrspace(1)* %out, <64 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <64 x i16>, <64 x i16> addrspace(1)* %in + %ext = zext <64 x i16> %load to <64 x i64> + store <64 x i64> %ext, <64 x i64> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}sextload_global_v64i16_to_v64i64: +; SI: s_endpgm +define void @sextload_global_v64i16_to_v64i64(<64 x i64> addrspace(1)* %out, <64 x i16> addrspace(1)* nocapture %in) nounwind { + %load = load <64 x i16>, <64 x i16> addrspace(1)* %in + %ext = sext <64 x i16> %load to <64 x i64> + store <64 x i64> %ext, <64 x i64> addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/global-variable-relocs.ll b/test/CodeGen/AMDGPU/global-variable-relocs.ll index c39394a3527d..00be6e4d5c15 100644 --- a/test/CodeGen/AMDGPU/global-variable-relocs.ll +++ b/test/CodeGen/AMDGPU/global-variable-relocs.ll @@ -14,8 +14,8 @@ ; CHECK-LABEL: private_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], private+8 -; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], private@rel32@lo+8 +; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], private@rel32@hi+8 ; CHECK-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[ADDR_LO]] ; CHECK-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], s[[ADDR_HI]] ; CHECK: flat_load_dword v{{[0-9]+}}, v{{\[}}[[V_LO]]:[[V_HI]]{{\]}} @@ -28,8 +28,8 @@ define void @private_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: internal_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], internal+8 -; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], internal@rel32@lo+8 +; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], internal@rel32@hi+8 ; CHECK-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[ADDR_LO]] ; CHECK-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], s[[ADDR_HI]] ; CHECK: flat_load_dword v{{[0-9]+}}, v{{\[}}[[V_LO]]:[[V_HI]]{{\]}} @@ -42,8 +42,8 @@ define void @internal_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: available_externally_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], available_externally@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], available_externally@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], available_externally@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -59,8 +59,8 @@ define void @available_externally_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: linkonce_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], linkonce@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], linkonce@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], linkonce@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -76,8 +76,8 @@ define void @linkonce_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: weak_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], weak@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], weak@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], weak@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -93,8 +93,8 @@ define void @weak_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: common_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], common@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], common@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], common@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -110,8 +110,8 @@ define void @common_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: extern_weak_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], extern_weak@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], extern_weak@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], extern_weak@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -127,8 +127,8 @@ define void @extern_weak_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: linkonce_odr_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], linkonce_odr@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], linkonce_odr@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], linkonce_odr@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -144,8 +144,8 @@ define void @linkonce_odr_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: weak_odr_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], weak_odr@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], weak_odr@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], weak_odr@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -161,8 +161,8 @@ define void @weak_odr_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: external_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], external@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], external@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], external@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 @@ -178,8 +178,8 @@ define void @external_test(i32 addrspace(1)* %out) { ; CHECK-LABEL: external_w_init_test: ; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}} -; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], external_w_init@GOTPCREL+4 -; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], 0 +; CHECK: s_add_u32 s[[GOTADDR_LO:[0-9]+]], s[[PC_LO]], external_w_init@gotpcrel32@lo+4 +; CHECK: s_addc_u32 s[[GOTADDR_HI:[0-9]+]], s[[PC_HI]], external_w_init@gotpcrel32@hi+4 ; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOTADDR_LO]]:[[GOTADDR_HI]]{{\]}}, 0x0 ; CHECK: s_add_u32 s[[GEP_LO:[0-9]+]], s[[ADDR_LO]], 4 ; CHECK: s_addc_u32 s[[GEP_HI:[0-9]+]], s[[ADDR_HI]], 0 diff --git a/test/CodeGen/AMDGPU/global_smrd.ll b/test/CodeGen/AMDGPU/global_smrd.ll new file mode 100644 index 000000000000..20890894c0ca --- /dev/null +++ b/test/CodeGen/AMDGPU/global_smrd.ll @@ -0,0 +1,126 @@ +; RUN: llc -O2 -mtriple amdgcn--amdhsa -mcpu=fiji -amdgpu-scalarize-global-loads=true -verify-machineinstrs < %s | FileCheck %s + +; uniform loads +; CHECK-LABEL: @uniform_load +; CHECK: s_load_dwordx4 +; CHECK-NOT: flat_load_dword + +define amdgpu_kernel void @uniform_load(float addrspace(1)* %arg, float addrspace(1)* %arg1) { +bb: + %tmp2 = load float, float addrspace(1)* %arg, align 4, !tbaa !8 + %tmp3 = fadd float %tmp2, 0.000000e+00 + %tmp4 = getelementptr inbounds float, float addrspace(1)* %arg, i64 1 + %tmp5 = load float, float addrspace(1)* %tmp4, align 4, !tbaa !8 + %tmp6 = fadd float %tmp3, %tmp5 + %tmp7 = getelementptr inbounds float, float addrspace(1)* %arg, i64 2 + %tmp8 = load float, float addrspace(1)* %tmp7, align 4, !tbaa !8 + %tmp9 = fadd float %tmp6, %tmp8 + %tmp10 = getelementptr inbounds float, float addrspace(1)* %arg, i64 3 + %tmp11 = load float, float addrspace(1)* %tmp10, align 4, !tbaa !8 + %tmp12 = fadd float %tmp9, %tmp11 + %tmp13 = getelementptr inbounds float, float addrspace(1)* %arg1 + store float %tmp12, float addrspace(1)* %tmp13, align 4, !tbaa !8 + ret void +} + +; non-uniform loads +; CHECK-LABEL: @non-uniform_load +; CHECK: flat_load_dword +; CHECK-NOT: s_load_dwordx4 + +define amdgpu_kernel void @non-uniform_load(float addrspace(1)* %arg, float addrspace(1)* %arg1) #0 { +bb: + %tmp = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tmp2 = getelementptr inbounds float, float addrspace(1)* %arg, i32 %tmp + %tmp3 = load float, float addrspace(1)* %tmp2, align 4, !tbaa !8 + %tmp4 = fadd float %tmp3, 0.000000e+00 + %tmp5 = add i32 %tmp, 1 + %tmp6 = getelementptr inbounds float, float addrspace(1)* %arg, i32 %tmp5 + %tmp7 = load float, float addrspace(1)* %tmp6, align 4, !tbaa !8 + %tmp8 = fadd float %tmp4, %tmp7 + %tmp9 = add i32 %tmp, 2 + %tmp10 = getelementptr inbounds float, float addrspace(1)* %arg, i32 %tmp9 + %tmp11 = load float, float addrspace(1)* %tmp10, align 4, !tbaa !8 + %tmp12 = fadd float %tmp8, %tmp11 + %tmp13 = add i32 %tmp, 3 + %tmp14 = getelementptr inbounds float, float addrspace(1)* %arg, i32 %tmp13 + %tmp15 = load float, float addrspace(1)* %tmp14, align 4, !tbaa !8 + %tmp16 = fadd float %tmp12, %tmp15 + %tmp17 = getelementptr inbounds float, float addrspace(1)* %arg1, i32 %tmp + store float %tmp16, float addrspace(1)* %tmp17, align 4, !tbaa !8 + ret void +} + + +; uniform load dominated by no-alias store - scalarize +; CHECK-LABEL: @no_memdep_alias_arg +; CHECK: flat_store_dword +; CHECK: s_load_dword [[SVAL:s[0-9]+]] +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]] +; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]] + +define amdgpu_kernel void @no_memdep_alias_arg(i32 addrspace(1)* noalias %in, i32 addrspace(1)* %out0, i32 addrspace(1)* %out1) { + store i32 0, i32 addrspace(1)* %out0 + %val = load i32, i32 addrspace(1)* %in + store i32 %val, i32 addrspace(1)* %out1 + ret void +} + +; uniform load dominated by alias store - vector +; CHECK-LABEL: {{^}}memdep: +; CHECK: flat_store_dword +; CHECK: flat_load_dword [[VVAL:v[0-9]+]] +; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]] +define amdgpu_kernel void @memdep(i32 addrspace(1)* %in, i32 addrspace(1)* %out0, i32 addrspace(1)* %out1) { + store i32 0, i32 addrspace(1)* %out0 + %val = load i32, i32 addrspace(1)* %in + store i32 %val, i32 addrspace(1)* %out1 + ret void +} + +; uniform load from global array +; CHECK-LABEL: @global_array +; CHECK: s_load_dwordx2 [[A_ADDR:s\[[0-9]+:[0-9]+\]]] +; CHECK: s_load_dwordx2 [[A_ADDR1:s\[[0-9]+:[0-9]+\]]], [[A_ADDR]], 0x0 +; CHECK: s_load_dword [[SVAL:s[0-9]+]], [[A_ADDR1]], 0x0 +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]] +; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]] + +@A = common local_unnamed_addr addrspace(1) global i32 addrspace(1)* null, align 4 + +define amdgpu_kernel void @global_array(i32 addrspace(1)* nocapture %out) { +entry: + %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* @A, align 4 + %1 = load i32, i32 addrspace(1)* %0, align 4 + store i32 %1, i32 addrspace(1)* %out, align 4 + ret void +} + + +; uniform load from global array dominated by alias store +; CHECK-LABEL: @global_array_alias_store +; CHECK: flat_store_dword +; CHECK: v_mov_b32_e32 v[[ADDR_LO:[0-9]+]], s{{[0-9]+}} +; CHECK: v_mov_b32_e32 v[[ADDR_HI:[0-9]+]], s{{[0-9]+}} +; CHECK: flat_load_dwordx2 [[A_ADDR:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[ADDR_LO]]:[[ADDR_HI]]{{\]}} +; CHECK: flat_load_dword [[VVAL:v[0-9]+]], [[A_ADDR]] +; CHECK: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[VVAL]] +define amdgpu_kernel void @global_array_alias_store(i32 addrspace(1)* nocapture %out, i32 %n) { +entry: + %gep = getelementptr i32, i32 addrspace(1) * %out, i32 %n + store i32 12, i32 addrspace(1) * %gep + %0 = load i32 addrspace(1)*, i32 addrspace(1)* addrspace(1)* @A, align 4 + %1 = load i32, i32 addrspace(1)* %0, align 4 + store i32 %1, i32 addrspace(1)* %out, align 4 + ret void +} + + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #1 = { nounwind readnone } + +!8 = !{!9, !9, i64 0} +!9 = !{!"float", !10, i64 0} +!10 = !{!"omnipotent char", !11, i64 0} +!11 = !{!"Simple C/C++ TBAA"} diff --git a/test/CodeGen/AMDGPU/global_smrd_cfg.ll b/test/CodeGen/AMDGPU/global_smrd_cfg.ll new file mode 100644 index 000000000000..a6a04151caa9 --- /dev/null +++ b/test/CodeGen/AMDGPU/global_smrd_cfg.ll @@ -0,0 +1,80 @@ +; RUN: llc -mtriple amdgcn--amdhsa -mcpu=fiji -amdgpu-scalarize-global-loads=true -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: %bb11 + +; Load from %arg in a Loop body has alias store + +; CHECK: flat_load_dword + +; CHECK-LABEL: %bb20 +; CHECK: flat_store_dword + +; ##################################################################### + +; CHECK-LABEL: %bb22 + +; Load from %arg has alias store in Loop + +; CHECK: flat_load_dword + +; ##################################################################### + +; Load from %arg1 has no-alias store in Loop - arg1[i+1] never alias arg1[i] + +; CHECK: s_load_dword + +define amdgpu_kernel void @cfg(i32 addrspace(1)* nocapture readonly %arg, i32 addrspace(1)* nocapture %arg1, i32 %arg2) #0 { +bb: + %tmp = sext i32 %arg2 to i64 + %tmp3 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp + %tmp4 = load i32, i32 addrspace(1)* %tmp3, align 4, !tbaa !0 + %tmp5 = icmp sgt i32 %tmp4, 0 + br i1 %tmp5, label %bb6, label %bb8 + +bb6: ; preds = %bb + br label %bb11 + +bb7: ; preds = %bb22 + br label %bb8 + +bb8: ; preds = %bb7, %bb + %tmp9 = phi i32 [ 0, %bb ], [ %tmp30, %bb7 ] + %tmp10 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp + store i32 %tmp9, i32 addrspace(1)* %tmp10, align 4, !tbaa !0 + ret void + +bb11: ; preds = %bb22, %bb6 + %tmp12 = phi i32 [ %tmp30, %bb22 ], [ 0, %bb6 ] + %tmp13 = phi i32 [ %tmp25, %bb22 ], [ 0, %bb6 ] + %tmp14 = srem i32 %tmp13, %arg2 + %tmp15 = sext i32 %tmp14 to i64 + %tmp16 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp15 + %tmp17 = load i32, i32 addrspace(1)* %tmp16, align 4, !tbaa !0 + %tmp18 = icmp sgt i32 %tmp17, 100 + %tmp19 = sext i32 %tmp13 to i64 + br i1 %tmp18, label %bb20, label %bb22 + +bb20: ; preds = %bb11 + %tmp21 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp19 + store i32 0, i32 addrspace(1)* %tmp21, align 4, !tbaa !0 + br label %bb22 + +bb22: ; preds = %bb20, %bb11 + %tmp23 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 %tmp19 + %tmp24 = load i32, i32 addrspace(1)* %tmp23, align 4, !tbaa !0 + %tmp25 = add nuw nsw i32 %tmp13, 1 + %tmp26 = sext i32 %tmp25 to i64 + %tmp27 = getelementptr inbounds i32, i32 addrspace(1)* %arg1, i64 %tmp26 + %tmp28 = load i32, i32 addrspace(1)* %tmp27, align 4, !tbaa !0 + %tmp29 = add i32 %tmp24, %tmp12 + %tmp30 = add i32 %tmp29, %tmp28 + %tmp31 = icmp eq i32 %tmp25, %tmp4 + br i1 %tmp31, label %bb7, label %bb11 +} + +attributes #0 = { "target-cpu"="fiji" } + +!0 = !{!1, !1, i64 0} +!1 = !{!"int", !2, i64 0} +!2 = !{!"omnipotent char", !3, i64 0} +!3 = !{!"Simple C/C++ TBAA"} diff --git a/test/CodeGen/AMDGPU/half.ll b/test/CodeGen/AMDGPU/half.ll index d21d66176a14..f2bb3f9d110a 100644 --- a/test/CodeGen/AMDGPU/half.ll +++ b/test/CodeGen/AMDGPU/half.ll @@ -1,11 +1,12 @@ ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s -; half args should be promoted to float +; half args should be promoted to float for SI and lower. ; GCN-LABEL: {{^}}load_f16_arg: ; GCN: s_load_dword [[ARG:s[0-9]+]] -; GCN: v_cvt_f16_f32_e32 [[CVT:v[0-9]+]], [[ARG]] +; SI: v_cvt_f16_f32_e32 [[CVT:v[0-9]+]], [[ARG]] +; VI: v_trunc_f16_e32 [[CVT:v[0-9]+]], [[ARG]] ; GCN: buffer_store_short [[CVT]] define void @load_f16_arg(half addrspace(1)* %out, half %arg) #0 { store half %arg, half addrspace(1)* %out @@ -16,7 +17,7 @@ define void @load_f16_arg(half addrspace(1)* %out, half %arg) #0 { ; GCN-DAG: buffer_load_ushort [[V0:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:44 ; GCN-DAG: buffer_load_ushort [[V1:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:46 ; GCN: v_lshlrev_b32_e32 [[HI:v[0-9]+]], 16, [[V1]] -; GCN: v_or_b32_e32 [[PACKED:v[0-9]+]], [[V0]], [[HI]] +; GCN: v_or_b32_e32 [[PACKED:v[0-9]+]], [[HI]], [[V0]] ; GCN: buffer_store_dword [[PACKED]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} ; GCN: s_endpgm define void @load_v2f16_arg(<2 x half> addrspace(1)* %out, <2 x half> %arg) #0 { @@ -131,8 +132,11 @@ define void @extload_v8f16_to_v8f32_arg(<8 x float> addrspace(1)* %out, <8 x hal ; GCN-LABEL: {{^}}extload_f16_to_f64_arg: ; SI: s_load_dword [[ARG:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb{{$}} +; SI: v_cvt_f64_f32_e32 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[ARG]] ; VI: s_load_dword [[ARG:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c{{$}} -; GCN: v_cvt_f64_f32_e32 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[ARG]] +; VI: v_trunc_f16_e32 v[[VARG:[0-9]+]], [[ARG]] +; VI: v_cvt_f32_f16_e32 v[[VARG_F32:[0-9]+]], v[[VARG]] +; VI: v_cvt_f64_f32_e32 [[RESULT:v\[[0-9]+:[0-9]+\]]], v[[VARG_F32]] ; GCN: buffer_store_dwordx2 [[RESULT]] define void @extload_f16_to_f64_arg(double addrspace(1)* %out, half %arg) #0 { %ext = fpext half %arg to double @@ -279,8 +283,9 @@ define void @global_extload_f16_to_f32(float addrspace(1)* %out, half addrspace( ; GCN-LABEL: {{^}}global_extload_v2f16_to_v2f32: ; GCN: buffer_load_dword [[LOAD:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; VI: v_lshrrev_b32_e32 [[HI:v[0-9]+]], 16, [[LOAD]] ; GCN: v_cvt_f32_f16_e32 v[[CVT0:[0-9]+]], [[LOAD]] -; GCN: v_lshrrev_b32_e32 [[HI:v[0-9]+]], 16, [[LOAD]] +; SI: v_lshrrev_b32_e32 [[HI:v[0-9]+]], 16, [[LOAD]] ; GCN: v_cvt_f32_f16_e32 v[[CVT1:[0-9]+]], [[HI]] ; GCN: buffer_store_dwordx2 v{{\[}}[[CVT0]]:[[CVT1]]{{\]}} ; GCN: s_endpgm @@ -379,19 +384,34 @@ define void @global_extload_v2f16_to_v2f64(<2 x double> addrspace(1)* %out, <2 x ; GCN-LABEL: {{^}}global_extload_v3f16_to_v3f64: -; GCN: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]] -; GCN-DAG: v_cvt_f32_f16_e32 -; GCN-DAG: v_lshrrev_b32_e32 {{v[0-9]+}}, 16, {{v[0-9]+}} -; GCN-DAG: v_cvt_f32_f16_e32 -; GCN-DAG: v_cvt_f32_f16_e32 +; XSI: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]] +; XSI: v_cvt_f32_f16_e32 +; XSI: v_cvt_f32_f16_e32 +; XSI-DAG: v_lshrrev_b32_e32 {{v[0-9]+}}, 16, {{v[0-9]+}} +; XSI: v_cvt_f32_f16_e32 +; XSI-NOT: v_cvt_f32_f16 + +; XVI: buffer_load_dwordx2 [[LOAD:v\[[0-9]+:[0-9]+\]]] +; XVI-DAG: v_lshrrev_b32_e32 {{v[0-9]+}}, 16, {{v[0-9]+}} +; XVI: v_cvt_f32_f16_e32 +; XVI: v_cvt_f32_f16_e32 +; XVI: v_cvt_f32_f16_e32 +; XVI-NOT: v_cvt_f32_f16 + +; GCN: buffer_load_dwordx2 v{{\[}}[[IN_LO:[0-9]+]]:[[IN_HI:[0-9]+]] +; VI: v_lshrrev_b32_e32 [[Y16:v[0-9]+]], 16, v[[IN_LO]] +; GCN: v_cvt_f32_f16_e32 [[Z32:v[0-9]+]], v[[IN_HI]] +; GCN: v_cvt_f32_f16_e32 [[X32:v[0-9]+]], v[[IN_LO]] +; SI: v_lshrrev_b32_e32 [[Y16:v[0-9]+]], 16, v[[IN_LO]] +; GCN: v_cvt_f32_f16_e32 [[Y32:v[0-9]+]], [[Y16]] -; GCN: v_cvt_f64_f32_e32 -; GCN: v_cvt_f64_f32_e32 -; GCN: v_cvt_f64_f32_e32 +; GCN: v_cvt_f64_f32_e32 [[Z:v\[[0-9]+:[0-9]+\]]], [[Z32]] +; GCN: v_cvt_f64_f32_e32 v{{\[}}[[XLO:[0-9]+]]:{{[0-9]+}}], [[X32]] +; GCN: v_cvt_f64_f32_e32 v[{{[0-9]+}}:[[YHI:[0-9]+]]{{\]}}, [[Y32]] ; GCN-NOT: v_cvt_f64_f32_e32 -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} -; GCN-DAG: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 +; GCN-DAG: buffer_store_dwordx4 v{{\[}}[[XLO]]:[[YHI]]{{\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}} +; GCN-DAG: buffer_store_dwordx2 [[Z]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:16 ; GCN: s_endpgm define void @global_extload_v3f16_to_v3f64(<3 x double> addrspace(1)* %out, <3 x half> addrspace(1)* %in) #0 { %val = load <3 x half>, <3 x half> addrspace(1)* %in @@ -440,7 +460,7 @@ define void @global_truncstore_f32_to_f16(half addrspace(1)* %out, float addrspa ; GCN-DAG: v_cvt_f16_f32_e32 [[CVT0:v[0-9]+]], v[[LO]] ; GCN-DAG: v_cvt_f16_f32_e32 [[CVT1:v[0-9]+]], v[[HI]] ; GCN-DAG: v_lshlrev_b32_e32 [[SHL:v[0-9]+]], 16, [[CVT1]] -; GCN-DAG: v_or_b32_e32 [[PACKED:v[0-9]+]], [[CVT0]], [[SHL]] +; GCN-DAG: v_or_b32_e32 [[PACKED:v[0-9]+]], [[SHL]], [[CVT0]] ; GCN-DAG: buffer_store_dword [[PACKED]] ; GCN: s_endpgm define void @global_truncstore_v2f32_to_v2f16(<2 x half> addrspace(1)* %out, <2 x float> addrspace(1)* %in) #0 { @@ -587,18 +607,6 @@ define void @fadd_v8f16(<8 x half> addrspace(1)* %out, <8 x half> %a, <8 x half> ret void } -; GCN-LABEL: {{^}}fsub_f16: -; GCN: v_subrev_f32_e32 -; GCN: s_endpgm -define void @fsub_f16(half addrspace(1)* %out, half addrspace(1)* %in) #0 { - %b_ptr = getelementptr half, half addrspace(1)* %in, i32 1 - %a = load half, half addrspace(1)* %in - %b = load half, half addrspace(1)* %b_ptr - %sub = fsub half %a, %b - store half %sub, half addrspace(1)* %out - ret void -} - ; GCN-LABEL: {{^}}test_bitcast_from_half: ; GCN: buffer_load_ushort [[TMP:v[0-9]+]] ; GCN: buffer_store_short [[TMP]] diff --git a/test/CodeGen/AMDGPU/hoist-cond.ll b/test/CodeGen/AMDGPU/hoist-cond.ll new file mode 100644 index 000000000000..6fe53e718f63 --- /dev/null +++ b/test/CodeGen/AMDGPU/hoist-cond.ll @@ -0,0 +1,46 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s + +; Check that invariant compare is hoisted out of the loop. +; At the same time condition shall not be serialized into a VGPR and deserialized later +; using another v_cmp + v_cndmask, but used directly in s_and_saveexec_b64. + +; CHECK: v_cmp_{{..}}_u32_e{{32|64}} [[COND:s\[[0-9]+:[0-9]+\]|vcc]] +; CHECK: BB0_1: +; CHECK-NOT: v_cmp +; CHECK_NOT: v_cndmask +; CHECK: s_and_saveexec_b64 s[{{[[0-9]+:[0-9]+}}], [[COND]] +; CHECK: BB0_2: + +define amdgpu_kernel void @hoist_cond(float addrspace(1)* nocapture %arg, float addrspace(1)* noalias nocapture readonly %arg1, i32 %arg3, i32 %arg4) { +bb: + %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 + %tmp5 = icmp ult i32 %tmp, %arg3 + br label %bb1 + +bb1: ; preds = %bb3, %bb + %tmp7 = phi i32 [ %arg4, %bb ], [ %tmp16, %bb3 ] + %tmp8 = phi float [ 0.000000e+00, %bb ], [ %tmp15, %bb3 ] + br i1 %tmp5, label %bb2, label %bb3 + +bb2: ; preds = %bb1 + %tmp10 = zext i32 %tmp7 to i64 + %tmp11 = getelementptr inbounds float, float addrspace(1)* %arg1, i64 %tmp10 + %tmp12 = load float, float addrspace(1)* %tmp11, align 4 + br label %bb3 + +bb3: ; preds = %bb2, %bb1 + %tmp14 = phi float [ %tmp12, %bb2 ], [ 0.000000e+00, %bb1 ] + %tmp15 = fadd float %tmp8, %tmp14 + %tmp16 = add i32 %tmp7, -1 + %tmp17 = icmp eq i32 %tmp16, 0 + br i1 %tmp17, label %bb4, label %bb1 + +bb4: ; preds = %bb3 + store float %tmp15, float addrspace(1)* %arg, align 4 + ret void +} + +; Function Attrs: nounwind readnone +declare i32 @llvm.amdgcn.workitem.id.x() #0 + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/hsa-fp-mode.ll b/test/CodeGen/AMDGPU/hsa-fp-mode.ll index 36aa6779d382..51d6aee25f45 100644 --- a/test/CodeGen/AMDGPU/hsa-fp-mode.ll +++ b/test/CodeGen/AMDGPU/hsa-fp-mode.ll @@ -1,9 +1,9 @@ ; RUN: llc -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; GCN-LABEL: {{^}}test_default_ci: -; GCN: compute_pgm_rsrc1_float_mode = 192 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 192 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_default_ci(float addrspace(1)* %out0, double addrspace(1)* %out1) #0 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -11,9 +11,9 @@ define void @test_default_ci(float addrspace(1)* %out0, double addrspace(1)* %ou } ; GCN-LABEL: {{^}}test_default_vi: -; GCN: compute_pgm_rsrc1_float_mode = 192 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 192 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %out1) #1 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -21,9 +21,9 @@ define void @test_default_vi(float addrspace(1)* %out0, double addrspace(1)* %ou } ; GCN-LABEL: {{^}}test_f64_denormals: -; GCN: compute_pgm_rsrc1_float_mode = 192 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 192 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #2 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -31,9 +31,9 @@ define void @test_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* } ; GCN-LABEL: {{^}}test_f32_denormals: -; GCN: compute_pgm_rsrc1_float_mode = 48 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 48 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_f32_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #3 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -41,9 +41,9 @@ define void @test_f32_denormals(float addrspace(1)* %out0, double addrspace(1)* } ; GCN-LABEL: {{^}}test_f32_f64_denormals: -; GCN: compute_pgm_rsrc1_float_mode = 240 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 240 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_f32_f64_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #4 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 @@ -51,9 +51,9 @@ define void @test_f32_f64_denormals(float addrspace(1)* %out0, double addrspace( } ; GCN-LABEL: {{^}}test_no_denormals: -; GCN: compute_pgm_rsrc1_float_mode = 0 -; GCN: compute_pgm_rsrc1_dx10_clamp = 1 -; GCN: compute_pgm_rsrc1_ieee_mode = 0 +; GCN: float_mode = 0 +; GCN: enable_dx10_clamp = 1 +; GCN: enable_ieee_mode = 1 define void @test_no_denormals(float addrspace(1)* %out0, double addrspace(1)* %out1) #5 { store float 0.0, float addrspace(1)* %out0 store double 0.0, double addrspace(1)* %out1 diff --git a/test/CodeGen/AMDGPU/hsa-globals.ll b/test/CodeGen/AMDGPU/hsa-globals.ll index df478fbcf3b5..2820b308edb8 100644 --- a/test/CodeGen/AMDGPU/hsa-globals.ll +++ b/test/CodeGen/AMDGPU/hsa-globals.ll @@ -38,7 +38,7 @@ define void @test() { ; ASM: .size external_global_program, 4 ; ASM: .type internal_readonly,@object -; ASM: .text +; ASM: .section .rodata.cst4,"aM",@progbits,4 ; ASM: internal_readonly: ; ASM: .long 0 ; ASM: .size internal_readonly, 4 diff --git a/test/CodeGen/AMDGPU/hsa-note-no-func.ll b/test/CodeGen/AMDGPU/hsa-note-no-func.ll index 1b4a0f3090b8..ae6c1891a84c 100644 --- a/test/CodeGen/AMDGPU/hsa-note-no-func.ll +++ b/test/CodeGen/AMDGPU/hsa-note-no-func.ll @@ -1,8 +1,26 @@ -; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA --check-prefix=HSA-CI %s -; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo | FileCheck --check-prefix=HSA --check-prefix=HSA-VI %s -; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji | FileCheck --check-prefix=HSA --check-prefix=HSA-FIJI %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx700 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI700 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx701 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI701 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx702 | FileCheck --check-prefix=HSA --check-prefix=HSA-CI702 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=kaveri | FileCheck --check-prefix=HSA --check-prefix=HSA-CI700 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=carrizo | FileCheck --check-prefix=HSA --check-prefix=HSA-VI801 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=tonga | FileCheck --check-prefix=HSA --check-prefix=HSA-VI802 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris10 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=polaris11 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx800 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI800 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx801 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI801 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx802 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI802 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx803 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI803 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx804 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI804 %s +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=gfx810 | FileCheck --check-prefix=HSA --check-prefix=HSA-VI810 %s ; HSA: .hsa_code_object_version 2,1 -; HSA-CI: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU" -; HSA-VI: .hsa_code_object_isa 8,0,1,"AMD","AMDGPU" -; HSA-FIJI: .hsa_code_object_isa 8,0,3,"AMD","AMDGPU" +; HSA-CI700: .hsa_code_object_isa 7,0,0,"AMD","AMDGPU" +; HSA-CI701: .hsa_code_object_isa 7,0,1,"AMD","AMDGPU" +; HSA-CI702: .hsa_code_object_isa 7,0,2,"AMD","AMDGPU" +; HSA-VI800: .hsa_code_object_isa 8,0,0,"AMD","AMDGPU" +; HSA-VI801: .hsa_code_object_isa 8,0,1,"AMD","AMDGPU" +; HSA-VI802: .hsa_code_object_isa 8,0,2,"AMD","AMDGPU" +; HSA-VI803: .hsa_code_object_isa 8,0,3,"AMD","AMDGPU" +; HSA-VI804: .hsa_code_object_isa 8,0,4,"AMD","AMDGPU" +; HSA-VI810: .hsa_code_object_isa 8,1,0,"AMD","AMDGPU" diff --git a/test/CodeGen/AMDGPU/hsa.ll b/test/CodeGen/AMDGPU/hsa.ll index 82d7da188ca3..78a5cdb576f5 100644 --- a/test/CodeGen/AMDGPU/hsa.ll +++ b/test/CodeGen/AMDGPU/hsa.ll @@ -17,11 +17,16 @@ ; ELF: } ; ELF: SHT_NOTE +; ELF: Flags [ (0x2) +; ELF: SHF_ALLOC (0x2) +; ELF: ] +; ELF: SectionData ( ; ELF: 0000: 04000000 08000000 01000000 414D4400 ; ELF: 0010: 02000000 01000000 04000000 1B000000 ; ELF: 0020: 03000000 414D4400 04000700 07000000 ; ELF: 0030: 00000000 00000000 414D4400 414D4447 ; ELF: 0040: 50550000 +; ELF: ) ; ELF: Symbol { ; ELF: Name: simple diff --git a/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll b/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll index d6309a2dd5de..e85db65e7429 100644 --- a/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll +++ b/test/CodeGen/AMDGPU/i1-copy-implicit-def.ll @@ -1,11 +1,10 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; SILowerI1Copies was not handling IMPLICIT_DEF ; SI-LABEL: {{^}}br_implicit_def: ; SI: BB#0: -; SI-NEXT: s_and_b64 vcc, exec -; SI-NEXT: s_cbranch_vccnz +; SI-NEXT: s_cbranch_scc1 define void @br_implicit_def(i32 addrspace(1)* %out, i32 %arg) #0 { bb: br i1 undef, label %bb1, label %bb2 diff --git a/test/CodeGen/AMDGPU/i1-copy-phi.ll b/test/CodeGen/AMDGPU/i1-copy-phi.ll index 4d50dc2f4023..fd9f469b057d 100644 --- a/test/CodeGen/AMDGPU/i1-copy-phi.ll +++ b/test/CodeGen/AMDGPU/i1-copy-phi.ll @@ -6,7 +6,7 @@ ; SI: s_and_saveexec_b64 ; SI: s_xor_b64 ; SI: v_mov_b32_e32 [[REG]], -1{{$}} -; SI: v_cmp_ne_i32_e32 vcc, 0, [[REG]] +; SI: v_cmp_ne_u32_e32 vcc, 0, [[REG]] ; SI: s_and_saveexec_b64 ; SI: s_xor_b64 ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/icmp.i16.ll b/test/CodeGen/AMDGPU/icmp.i16.ll new file mode 100644 index 000000000000..c3dad2d32033 --- /dev/null +++ b/test/CodeGen/AMDGPU/icmp.i16.ll @@ -0,0 +1,353 @@ +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=GCN -check-prefix=SI %s + +;;;==========================================================================;;; +;; 16-bit integer comparisons +;;;==========================================================================;;; + +; GCN-LABEL: {{^}}i16_eq: +; VI: v_cmp_eq_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_eq_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_eq(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp eq i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ne: +; VI: v_cmp_ne_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ne_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ne(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp ne i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ugt: +; VI: v_cmp_gt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_gt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ugt(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp ugt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_uge: +; VI: v_cmp_ge_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ge_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_uge(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp uge i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ult: +; VI: v_cmp_lt_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_lt_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ult(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp ult i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ule: +; VI: v_cmp_le_u16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_le_u32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ule(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp ule i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void + +} + +; GCN-LABEL: {{^}}i16_sgt: +; VI: v_cmp_gt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_gt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sgt(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp sgt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_sge: +; VI: v_cmp_ge_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ge_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sge(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp sge i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_slt: +; VI: v_cmp_lt_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_lt_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_slt(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp slt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_sle: +; VI: v_cmp_le_i16_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_le_i32_e32 vcc, v{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sle(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 addrspace(1)* %b.ptr) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %b.gep = getelementptr inbounds i16, i16 addrspace(1)* %b.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %b = load i16, i16 addrspace(1)* %b.gep + %tmp0 = icmp sle i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; These should be commuted to reduce code size +; GCN-LABEL: {{^}}i16_eq_v_s: +; VI: v_cmp_eq_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_eq_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_eq_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp eq i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ne_v_s: +; VI: v_cmp_ne_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ne_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ne_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp ne i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ugt_v_s: +; VI: v_cmp_lt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_lt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ugt_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp ugt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_uge_v_s: +; VI: v_cmp_le_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_uge_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp uge i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ult_v_s: +; VI: v_cmp_gt_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ult_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp ult i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_ule_v_s: +; VI: v_cmp_ge_u16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ge_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_ule_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp ule i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_sgt_v_s: +; VI: v_cmp_lt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_lt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sgt_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp sgt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_sge_v_s: +; VI: v_cmp_le_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sge_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp sge i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_slt_v_s: +; VI: v_cmp_gt_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_gt_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_slt_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp slt i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}i16_sle_v_s: +; VI: v_cmp_ge_i16_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; SI: v_cmp_ge_i32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @i16_sle_v_s(i32 addrspace(1)* %out, i16 addrspace(1)* %a.ptr, i16 %b) #0 { +entry: + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %tid.ext = sext i32 %tid to i64 + %a.gep = getelementptr inbounds i16, i16 addrspace(1)* %a.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %a = load i16, i16 addrspace(1)* %a.gep + %tmp0 = icmp sle i16 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out.gep + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/icmp64.ll b/test/CodeGen/AMDGPU/icmp64.ll index 0eaa33ebafed..3d42eac44297 100644 --- a/test/CodeGen/AMDGPU/icmp64.ll +++ b/test/CodeGen/AMDGPU/icmp64.ll @@ -2,7 +2,7 @@ ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; SI-LABEL: {{^}}test_i64_eq: -; SI: v_cmp_eq_i64 +; SI: v_cmp_eq_u64 define void @test_i64_eq(i32 addrspace(1)* %out, i64 %a, i64 %b) nounwind { %cmp = icmp eq i64 %a, %b %result = sext i1 %cmp to i32 @@ -11,7 +11,7 @@ define void @test_i64_eq(i32 addrspace(1)* %out, i64 %a, i64 %b) nounwind { } ; SI-LABEL: {{^}}test_i64_ne: -; SI: v_cmp_ne_i64 +; SI: v_cmp_ne_u64 define void @test_i64_ne(i32 addrspace(1)* %out, i64 %a, i64 %b) nounwind { %cmp = icmp ne i64 %a, %b %result = sext i1 %cmp to i32 diff --git a/test/CodeGen/AMDGPU/imm.ll b/test/CodeGen/AMDGPU/imm.ll index 674eceee8122..0412c5da4d53 100644 --- a/test/CodeGen/AMDGPU/imm.ll +++ b/test/CodeGen/AMDGPU/imm.ll @@ -1,10 +1,10 @@ -; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=CHECK %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=CHECK %s +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s ; Use a 64-bit value with lo bits that can be represented as an inline constant -; CHECK-LABEL: {{^}}i64_imm_inline_lo: -; CHECK: v_mov_b32_e32 v[[LO_VGPR:[0-9]+]], 5 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VGPR]]: +; GCN-LABEL: {{^}}i64_imm_inline_lo: +; GCN: v_mov_b32_e32 v[[LO_VGPR:[0-9]+]], 5 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VGPR]]: define void @i64_imm_inline_lo(i64 addrspace(1) *%out) { entry: store i64 1311768464867721221, i64 addrspace(1) *%out ; 0x1234567800000005 @@ -12,214 +12,232 @@ entry: } ; Use a 64-bit value with hi bits that can be represented as an inline constant -; CHECK-LABEL: {{^}}i64_imm_inline_hi: -; CHECK: v_mov_b32_e32 v[[HI_VGPR:[0-9]+]], 5 -; CHECK: buffer_store_dwordx2 v{{\[[0-9]+:}}[[HI_VGPR]] +; GCN-LABEL: {{^}}i64_imm_inline_hi: +; GCN: v_mov_b32_e32 v[[HI_VGPR:[0-9]+]], 5 +; GCN: buffer_store_dwordx2 v{{\[[0-9]+:}}[[HI_VGPR]] define void @i64_imm_inline_hi(i64 addrspace(1) *%out) { entry: store i64 21780256376, i64 addrspace(1) *%out ; 0x0000000512345678 ret void } -; CHECK-LABEL: {{^}}store_imm_neg_0.0_i64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_bfrev_b32_e32 v[[HI_VREG:[0-9]+]], 1{{$}} -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_imm_neg_0.0_i64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_bfrev_b32_e32 v[[HI_VREG:[0-9]+]], 1{{$}} +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_imm_neg_0.0_i64(i64 addrspace(1) *%out) { store i64 -9223372036854775808, i64 addrspace(1) *%out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_neg_0.0_i32: -; CHECK: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_neg_0.0_i32: +; GCN: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_neg_0.0_i32(i32 addrspace(1)* %out) { store i32 -2147483648, i32 addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_0.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_0.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_0.0_f32(float addrspace(1)* %out) { store float 0.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_imm_neg_0.0_f32: -; CHECK: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_imm_neg_0.0_f32: +; GCN: v_bfrev_b32_e32 [[REG:v[0-9]+]], 1{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_imm_neg_0.0_f32(float addrspace(1)* %out) { store float -0.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_0.5_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 0.5{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_0.5_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0.5{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_0.5_f32(float addrspace(1)* %out) { store float 0.5, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_0.5_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], -0.5{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_m_0.5_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], -0.5{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_m_0.5_f32(float addrspace(1)* %out) { store float -0.5, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_1.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 1.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_1.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 1.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_1.0_f32(float addrspace(1)* %out) { store float 1.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_1.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], -1.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_m_1.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], -1.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_m_1.0_f32(float addrspace(1)* %out) { store float -1.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_2.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 2.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_2.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 2.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_2.0_f32(float addrspace(1)* %out) { store float 2.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_2.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], -2.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_m_2.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], -2.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_m_2.0_f32(float addrspace(1)* %out) { store float -2.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_4.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 4.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_4.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 4.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_4.0_f32(float addrspace(1)* %out) { store float 4.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_4.0_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], -4.0{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}store_inline_imm_m_4.0_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], -4.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @store_inline_imm_m_4.0_f32(float addrspace(1)* %out) { store float -4.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_literal_imm_f32: -; CHECK: v_mov_b32_e32 [[REG:v[0-9]+]], 0x45800000 -; CHECK: buffer_store_dword [[REG]] + +; GCN-LABEL: {{^}}store_inline_imm_inv_2pi_f32: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3e22f983{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0.15915494{{$}} +; GCN: buffer_store_dword [[REG]] +define void @store_inline_imm_inv_2pi_f32(float addrspace(1)* %out) { + store float 0x3FC45F3060000000, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_inv_2pi_f32: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xbe22f983{{$}} +; GCN: buffer_store_dword [[REG]] +define void @store_inline_imm_m_inv_2pi_f32(float addrspace(1)* %out) { + store float 0xBFC45F3060000000, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_literal_imm_f32: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x45800000 +; GCN: buffer_store_dword [[REG]] define void @store_literal_imm_f32(float addrspace(1)* %out) { store float 4096.0, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_0.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_0.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_0.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_0.5_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 0.5, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_0.5_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 0.5{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_0.5_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0.5 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_0.5_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -0.5, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_0.5_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -0.5{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_0.5_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, -0.5 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_1.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 1.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_1.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 1.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_1.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 1.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_1.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -1.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_1.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -1.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_1.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, -1.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_2.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 2.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_2.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 2.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_2.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 2.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_2.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -2.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_2.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -2.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_2.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, -2.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_4.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 4.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_4.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 4.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_4.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 4.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_4.0_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -4.0, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_4.0_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -4.0{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_4.0_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, -4.0 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}commute_add_inline_imm_0.5_f32: -; CHECK: buffer_load_dword [[VAL:v[0-9]+]] -; CHECK: v_add_f32_e32 [[REG:v[0-9]+]], 0.5, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}commute_add_inline_imm_0.5_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN: v_add_f32_e32 [[REG:v[0-9]+]], 0.5, [[VAL]] +; GCN: buffer_store_dword [[REG]] define void @commute_add_inline_imm_0.5_f32(float addrspace(1)* %out, float addrspace(1)* %in) { %x = load float, float addrspace(1)* %in %y = fadd float %x, 0.5 @@ -227,10 +245,10 @@ define void @commute_add_inline_imm_0.5_f32(float addrspace(1)* %out, float addr ret void } -; CHECK-LABEL: {{^}}commute_add_literal_f32: -; CHECK: buffer_load_dword [[VAL:v[0-9]+]] -; CHECK: v_add_f32_e32 [[REG:v[0-9]+]], 0x44800000, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}commute_add_literal_f32: +; GCN: buffer_load_dword [[VAL:v[0-9]+]] +; GCN: v_add_f32_e32 [[REG:v[0-9]+]], 0x44800000, [[VAL]] +; GCN: buffer_store_dword [[REG]] define void @commute_add_literal_f32(float addrspace(1)* %out, float addrspace(1)* %in) { %x = load float, float addrspace(1)* %in %y = fadd float %x, 1024.0 @@ -238,80 +256,80 @@ define void @commute_add_literal_f32(float addrspace(1)* %out, float addrspace(1 ret void } -; CHECK-LABEL: {{^}}add_inline_imm_1_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 1, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_1_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 1{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_1_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0x36a0000000000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_2_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 2, [[VAL]]{{$}} -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_2_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 2{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_2_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0x36b0000000000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_16_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 16, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_16_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 16 +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_16_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0x36e0000000000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_1_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -1, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_1_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -1{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_1_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0xffffffffe0000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_2_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -2, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_2_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -2{{$}} +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_2_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0xffffffffc0000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_16_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], -16, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_neg_16_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], -16 +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_neg_16_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0xfffffffe00000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_63_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 63, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_63_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 63 +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_63_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0x36ff800000000000 store float %y, float addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_64_f32: -; CHECK: s_load_dword [[VAL:s[0-9]+]] -; CHECK: v_add_f32_e64 [[REG:v[0-9]+]], 64, [[VAL]] -; CHECK: buffer_store_dword [[REG]] +; GCN-LABEL: {{^}}add_inline_imm_64_f32: +; GCN: s_load_dword [[VAL:s[0-9]+]] +; GCN: v_add_f32_e64 [[REG:v[0-9]+]], [[VAL]], 64 +; GCN: buffer_store_dword [[REG]] define void @add_inline_imm_64_f32(float addrspace(1)* %out, float %x) { %y = fadd float %x, 0x3700000000000000 store float %y, float addrspace(1)* %out @@ -319,188 +337,212 @@ define void @add_inline_imm_64_f32(float addrspace(1)* %out, float %x) { } -; CHECK-LABEL: {{^}}add_inline_imm_0.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_0.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0{{$}} -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0{{$}} +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_0.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_0.5_f64: +; GCN-LABEL: {{^}}add_inline_imm_0.5_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0.5 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0.5 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_0.5_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0.5 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_0.5_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_0.5_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -0.5 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -0.5 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_0.5_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, -0.5 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_1.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_1.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 1.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 1.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_1.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 1.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_1.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_1.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -1.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -1.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_1.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, -1.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_2.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_2.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 2.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 2.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_2.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 2.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_2.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_2.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -2.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -2.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_2.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, -2.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_4.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_4.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 4.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 4.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_4.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 4.0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_4.0_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_4.0_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -4.0 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -4.0 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_4.0_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, -4.0 store double %y, double addrspace(1)* %out ret void } +; GCN-LABEL: {{^}}add_inline_imm_inv_2pi_f64: +; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb +; SI-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0x6dc9c882 +; SI-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3fc45f30 +; SI: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} -; CHECK-LABEL: {{^}}add_inline_imm_1_f64: +; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c +; VI: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0.15915494{{$}} +; VI: buffer_store_dwordx2 [[REG]] +define void @add_inline_imm_inv_2pi_f64(double addrspace(1)* %out, double %x) { + %y = fadd double %x, 0x3fc45f306dc9c882 + store double %y, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_m_inv_2pi_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0x6dc9c882 +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbfc45f30 +; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +define void @add_m_inv_2pi_f64(double addrspace(1)* %out, double %x) { + %y = fadd double %x, 0xbfc45f306dc9c882 + store double %y, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_1_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 1{{$}} -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 1{{$}} +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_1_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0x0000000000000001 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_2_f64: +; GCN-LABEL: {{^}}add_inline_imm_2_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 2{{$}} -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 2{{$}} +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_2_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0x0000000000000002 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_16_f64: +; GCN-LABEL: {{^}}add_inline_imm_16_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 16 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 16 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_16_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0x0000000000000010 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_1_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_1_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -1 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -1 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_1_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0xffffffffffffffff store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_2_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_2_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -2 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -2 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_2_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0xfffffffffffffffe store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_neg_16_f64: +; GCN-LABEL: {{^}}add_inline_imm_neg_16_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -16 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], -16 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_neg_16_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0xfffffffffffffff0 store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_63_f64: +; GCN-LABEL: {{^}}add_inline_imm_63_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 63 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 63 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_63_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0x000000000000003F store double %y, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}add_inline_imm_64_f64: +; GCN-LABEL: {{^}}add_inline_imm_64_f64: ; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x2c -; CHECK: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 64 -; CHECK: buffer_store_dwordx2 [[REG]] +; GCN: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 64 +; GCN: buffer_store_dwordx2 [[REG]] define void @add_inline_imm_64_f64(double addrspace(1)* %out, double %x) { %y = fadd double %x, 0x0000000000000040 store double %y, double addrspace(1)* %out @@ -508,101 +550,119 @@ define void @add_inline_imm_64_f64(double addrspace(1)* %out, double %x) { } -; CHECK-LABEL: {{^}}store_inline_imm_0.0_f64: -; CHECK: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0 -; CHECK: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], v[[LO_VREG]]{{$}} -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_0.0_f64: +; GCN: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0 +; GCN: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], v[[LO_VREG]]{{$}} +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_0.0_f64(double addrspace(1)* %out) { store double 0.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_literal_imm_neg_0.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_bfrev_b32_e32 v[[HI_VREG:[0-9]+]], 1{{$}} -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_literal_imm_neg_0.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_bfrev_b32_e32 v[[HI_VREG:[0-9]+]], 1{{$}} +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_literal_imm_neg_0.0_f64(double addrspace(1)* %out) { store double -0.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_0.5_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3fe00000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_0.5_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3fe00000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_0.5_f64(double addrspace(1)* %out) { store double 0.5, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_0.5_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbfe00000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_m_0.5_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbfe00000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_m_0.5_f64(double addrspace(1)* %out) { store double -0.5, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_1.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3ff00000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_1.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3ff00000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_1.0_f64(double addrspace(1)* %out) { store double 1.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_1.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbff00000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_m_1.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbff00000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_m_1.0_f64(double addrspace(1)* %out) { store double -1.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_2.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 2.0 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_2.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 2.0 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_2.0_f64(double addrspace(1)* %out) { store double 2.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_2.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], -2.0 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_m_2.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], -2.0 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_m_2.0_f64(double addrspace(1)* %out) { store double -2.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_4.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x40100000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_4.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x40100000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_4.0_f64(double addrspace(1)* %out) { store double 4.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_inline_imm_m_4.0_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xc0100000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inline_imm_m_4.0_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xc0100000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_inline_imm_m_4.0_f64(double addrspace(1)* %out) { store double -4.0, double addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}store_literal_imm_f64: -; CHECK-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} -; CHECK-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x40b00000 -; CHECK: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +; GCN-LABEL: {{^}}store_inv_2pi_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0x6dc9c882 +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x3fc45f30 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +define void @store_inv_2pi_f64(double addrspace(1)* %out) { + store double 0x3fc45f306dc9c882, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_inv_2pi_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0x6dc9c882 +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0xbfc45f30 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} +define void @store_inline_imm_m_inv_2pi_f64(double addrspace(1)* %out) { + store double 0xbfc45f306dc9c882, double addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_literal_imm_f64: +; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], 0{{$}} +; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], 0x40b00000 +; GCN: buffer_store_dwordx2 v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}} define void @store_literal_imm_f64(double addrspace(1)* %out) { store double 4096.0, double addrspace(1)* %out ret void diff --git a/test/CodeGen/AMDGPU/imm16.ll b/test/CodeGen/AMDGPU/imm16.ll new file mode 100644 index 000000000000..ed970287abbf --- /dev/null +++ b/test/CodeGen/AMDGPU/imm16.ll @@ -0,0 +1,316 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s + +; FIXME: Merge into imm.ll + +; GCN-LABEL: {{^}}store_inline_imm_neg_0.0_i16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0x8000{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffff8000{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_neg_0.0_i16(i16 addrspace(1)* %out) { + store volatile i16 -32768, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_0.0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_0.0_f16(half addrspace(1)* %out) { + store half 0.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_imm_neg_0.0_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0x8000{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffff8000{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_imm_neg_0.0_f16(half addrspace(1)* %out) { + store half -0.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_0.5_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3800{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_0.5_f16(half addrspace(1)* %out) { + store half 0.5, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_0.5_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xb800{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffb800{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_m_0.5_f16(half addrspace(1)* %out) { + store half -0.5, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_1.0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3c00{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_1.0_f16(half addrspace(1)* %out) { + store half 1.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_1.0_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xbc00{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffbc00{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_m_1.0_f16(half addrspace(1)* %out) { + store half -1.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_2.0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x4000{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_2.0_f16(half addrspace(1)* %out) { + store half 2.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_2.0_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xc000{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffc000{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_m_2.0_f16(half addrspace(1)* %out) { + store half -2.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_4.0_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x4400{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_4.0_f16(half addrspace(1)* %out) { + store half 4.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_4.0_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xc400{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffc400{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_m_4.0_f16(half addrspace(1)* %out) { + store half -4.0, half addrspace(1)* %out + ret void +} + + +; GCN-LABEL: {{^}}store_inline_imm_inv_2pi_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x3118{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_inv_2pi_f16(half addrspace(1)* %out) { + store half 0xH3118, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_inline_imm_m_inv_2pi_f16: +; SI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xb118{{$}} +; VI: v_mov_b32_e32 [[REG:v[0-9]+]], 0xffffb118{{$}} +; GCN: buffer_store_short [[REG]] +define void @store_inline_imm_m_inv_2pi_f16(half addrspace(1)* %out) { + store half 0xHB118, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}store_literal_imm_f16: +; GCN: v_mov_b32_e32 [[REG:v[0-9]+]], 0x6c00 +; GCN: buffer_store_short [[REG]] +define void @store_literal_imm_f16(half addrspace(1)* %out) { + store half 4096.0, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_0.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_0.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_0.5_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 0.5, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_0.5_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0.5 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_0.5_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -0.5, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_0.5_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, -0.5 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_1.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 1.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_1.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 1.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_1.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -1.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_1.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, -1.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_2.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 2.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_2.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 2.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_2.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -2.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_2.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, -2.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_4.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 4.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_4.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 4.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_4.0_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -4.0, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_4.0_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, -4.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_add_inline_imm_0.5_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 0.5, [[VAL]] +; VI: buffer_store_short [[REG]] +define void @commute_add_inline_imm_0.5_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %x = load half, half addrspace(1)* %in + %y = fadd half %x, 0.5 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_add_literal_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 0x6400, [[VAL]] +; VI: buffer_store_short [[REG]] +define void @commute_add_literal_f16(half addrspace(1)* %out, half addrspace(1)* %in) { + %x = load half, half addrspace(1)* %in + %y = fadd half %x, 1024.0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_1_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 1, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_1_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xH0001 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_2_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 2, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_2_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xH0002 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_16_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 16, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_16_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xH0010 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_1_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -1, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_1_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xHFFFF + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_2_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -2, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_2_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xHFFFE + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_neg_16_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], -16, [[VAL]]{{$}} +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_neg_16_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xHFFF0 + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_63_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 63, [[VAL]] +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_63_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xH003F + store half %y, half addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}add_inline_imm_64_f16: +; VI: buffer_load_ushort [[VAL:v[0-9]+]] +; VI: v_add_f16_e32 [[REG:v[0-9]+]], 64, [[VAL]] +; VI: buffer_store_short [[REG]] +define void @add_inline_imm_64_f16(half addrspace(1)* %out, half %x) { + %y = fadd half %x, 0xH0040 + store half %y, half addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll b/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll new file mode 100644 index 000000000000..877956be3088 --- /dev/null +++ b/test/CodeGen/AMDGPU/indirect-addressing-si-noopt.ll @@ -0,0 +1,19 @@ +; RUN: llc -O0 -march=amdgcn -verify-machineinstrs < %s | FileCheck %s + +; FIXME: Merge into indirect-addressing-si.ll + +; Make sure that TwoAddressInstructions keeps src0 as subregister sub0 +; of the tied implicit use and def of the super register. + +; CHECK-LABEL: {{^}}insert_wo_offset: +; CHECK: s_load_dword [[IN:s[0-9]+]] +; CHECK: s_mov_b32 m0, [[IN]] +; CHECK: v_movreld_b32_e32 v[[ELT0:[0-9]+]] +; CHECK-NEXT: buffer_store_dwordx4 v{{\[}}[[ELT0]]: +define void @insert_wo_offset(<4 x float> addrspace(1)* %out, i32 %in) { +entry: + %ins = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %in + store <4 x float> %ins, <4 x float> addrspace(1)* %out + ret void +} + diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/test/CodeGen/AMDGPU/indirect-addressing-si.ll index 66cec88e760c..528e12b76ce0 100644 --- a/test/CodeGen/AMDGPU/indirect-addressing-si.ll +++ b/test/CodeGen/AMDGPU/indirect-addressing-si.ll @@ -1,16 +1,23 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=MOVREL %s +; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-vgpr-index-mode -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=IDXMODE %s ; Tests for indirect addressing on SI, which is implemented using dynamic ; indexing of vectors. -; CHECK-LABEL: {{^}}extract_w_offset: -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 1.0 -; CHECK: s_mov_b32 m0 -; CHECK-NEXT: v_movrels_b32_e32 +; GCN-LABEL: {{^}}extract_w_offset: +; GCN-DAG: s_load_dword [[IN:s[0-9]+]] +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 +; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 1.0 + +; MOVREL-DAG: s_mov_b32 m0, [[IN]] +; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, [[BASEREG]] + +; IDXMODE: s_set_gpr_idx_on [[IN]], src0{{$}} +; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, [[BASEREG]] +; IDXMODE-NEXT: s_set_gpr_idx_off define void @extract_w_offset(float addrspace(1)* %out, i32 %in) { entry: %idx = add i32 %in, 1 @@ -20,17 +27,22 @@ entry: } ; XXX: Could do v_or_b32 directly -; CHECK-LABEL: {{^}}extract_w_offset_salu_use_vector: -; CHECK-DAG: s_or_b32 -; CHECK-DAG: s_or_b32 -; CHECK-DAG: s_or_b32 -; CHECK-DAG: s_or_b32 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} -; CHECK: s_mov_b32 m0 -; CHECK-NEXT: v_movrels_b32_e32 +; GCN-LABEL: {{^}}extract_w_offset_salu_use_vector: +; MOVREL: s_mov_b32 m0 +; GCN-DAG: s_or_b32 +; GCN-DAG: s_or_b32 +; GCN-DAG: s_or_b32 +; GCN-DAG: s_or_b32 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} + +; MOVREL: v_movrels_b32_e32 + +; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, src0{{$}} +; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE-NEXT: s_set_gpr_idx_off define void @extract_w_offset_salu_use_vector(i32 addrspace(1)* %out, i32 %in, <4 x i32> %or.val) { entry: %idx = add i32 %in, 1 @@ -40,13 +52,19 @@ entry: ret void } -; CHECK-LABEL: {{^}}extract_wo_offset: -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0 -; CHECK-DAG: v_mov_b32_e32 v{{[0-9]+}}, 1.0 -; CHECK: s_mov_b32 m0 -; CHECK-NEXT: v_movrels_b32_e32 +; GCN-LABEL: {{^}}extract_wo_offset: +; GCN-DAG: s_load_dword [[IN:s[0-9]+]] +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0 +; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 1.0 + +; MOVREL-DAG: s_mov_b32 m0, [[IN]] +; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, [[BASEREG]] + +; IDXMODE: s_set_gpr_idx_on [[IN]], src0{{$}} +; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, [[BASEREG]] +; IDXMODE-NEXT: s_set_gpr_idx_off define void @extract_wo_offset(float addrspace(1)* %out, i32 %in) { entry: %elt = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %in @@ -54,10 +72,17 @@ entry: ret void } -; CHECK-LABEL: {{^}}extract_neg_offset_sgpr: +; GCN-LABEL: {{^}}extract_neg_offset_sgpr: ; The offset depends on the register that holds the first element of the vector. -; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} -; CHECK: v_movrels_b32_e32 v{{[0-9]}}, v0 +; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} +; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} +; IDXMODE: v_mov_b32_e32 v2, 2 +; IDXMODE: v_mov_b32_e32 v3, 3 +; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}} +; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE-NEXT: s_set_gpr_idx_off define void @extract_neg_offset_sgpr(i32 addrspace(1)* %out, i32 %offset) { entry: %index = add i32 %offset, -512 @@ -66,10 +91,19 @@ entry: ret void } -; CHECK-LABEL: {{^}}extract_neg_offset_sgpr_loaded: +; GCN-LABEL: {{^}}extract_neg_offset_sgpr_loaded: ; The offset depends on the register that holds the first element of the vector. -; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} -; CHECK: v_movrels_b32_e32 v{{[0-9]}}, v0 +; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} +; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} +; IDXMODE: v_mov_b32_e32 v0, +; IDXMODE: v_mov_b32_e32 v1, +; IDXMODE: v_mov_b32_e32 v2, +; IDXMODE: v_mov_b32_e32 v3, +; IDXMODE-NEXT: s_set_gpr_idx_on [[ADD_IDX]], src0{{$}} +; IDXMODE-NEXT: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE-NEXT: s_set_gpr_idx_off define void @extract_neg_offset_sgpr_loaded(i32 addrspace(1)* %out, <4 x i32> %vec0, <4 x i32> %vec1, i32 %offset) { entry: %index = add i32 %offset, -512 @@ -79,12 +113,29 @@ entry: ret void } -; CHECK-LABEL: {{^}}extract_neg_offset_vgpr: +; GCN-LABEL: {{^}}extract_neg_offset_vgpr: ; The offset depends on the register that holds the first element of the vector. -; CHECK: v_readfirstlane_b32 -; CHECK: s_add_i32 m0, m0, 0xfffffe{{[0-9a-z]+}} -; CHECK-NEXT: v_movrels_b32_e32 v{{[0-9]}}, v0 -; CHECK: s_cbranch_execnz + +; FIXME: The waitcnt for the argument load can go after the loop +; IDXMODE: s_set_gpr_idx_on 0, src0 +; GCN: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec +; GCN: s_waitcnt lgkmcnt(0) + +; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v{{[0-9]+}} + +; MOVREL: s_add_i32 m0, [[READLANE]], 0xfffffe0 +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL: v_movrels_b32_e32 [[RESULT:v[0-9]+]], v1 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00 +; IDXMODE: s_set_gpr_idx_idx [[ADD_IDX]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE: v_mov_b32_e32 [[RESULT:v[0-9]+]], v1 + +; GCN: s_cbranch_execnz + +; IDXMODE: s_set_gpr_idx_off +; GCN: buffer_store_dword [[RESULT]] define void @extract_neg_offset_vgpr(i32 addrspace(1)* %out) { entry: %id = call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -94,7 +145,7 @@ entry: ret void } -; CHECK-LABEL: {{^}}extract_undef_offset_sgpr: +; GCN-LABEL: {{^}}extract_undef_offset_sgpr: define void @extract_undef_offset_sgpr(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { entry: %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in @@ -103,10 +154,10 @@ entry: ret void } -; CHECK-LABEL: {{^}}insert_undef_offset_sgpr_vector_src: -; CHECK: buffer_load_dwordx4 -; CHECK: s_mov_b32 m0, -; CHECK-NEXT: v_movreld_b32 +; GCN-LABEL: {{^}}insert_undef_offset_sgpr_vector_src: +; GCN-DAG: buffer_load_dwordx4 +; MOVREL-DAG: s_mov_b32 m0, +; MOVREL: v_movreld_b32 define void @insert_undef_offset_sgpr_vector_src(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { entry: %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in @@ -115,33 +166,52 @@ entry: ret void } -; CHECK-LABEL: {{^}}insert_w_offset: -; CHECK: s_mov_b32 m0 -; CHECK-NEXT: v_movreld_b32_e32 -define void @insert_w_offset(float addrspace(1)* %out, i32 %in) { +; GCN-LABEL: {{^}}insert_w_offset: +; GCN-DAG: s_load_dword [[IN:s[0-9]+]] +; MOVREL-DAG: s_mov_b32 m0, [[IN]] +; GCN-DAG: v_mov_b32_e32 v[[ELT0:[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 v[[ELT1:[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 v[[ELT2:[0-9]+]], 0x40400000 +; GCN-DAG: v_mov_b32_e32 v[[ELT3:[0-9]+]], 4.0 +; GCN-DAG: v_mov_b32_e32 v[[INS:[0-9]+]], 0x40a00000 + +; MOVREL: v_movreld_b32_e32 v[[ELT1]], v[[INS]] +; MOVREL: buffer_store_dwordx4 v{{\[}}[[ELT0]]:[[ELT3]]{{\]}} +define void @insert_w_offset(<4 x float> addrspace(1)* %out, i32 %in) { entry: %0 = add i32 %in, 1 %1 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %0 - %2 = extractelement <4 x float> %1, i32 2 - store float %2, float addrspace(1)* %out + store <4 x float> %1, <4 x float> addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}insert_wo_offset: -; CHECK: s_mov_b32 m0 -; CHECK-NEXT: v_movreld_b32_e32 -define void @insert_wo_offset(float addrspace(1)* %out, i32 %in) { +; GCN-LABEL: {{^}}insert_wo_offset: +; GCN: s_load_dword [[IN:s[0-9]+]] + +; MOVREL: s_mov_b32 m0, [[IN]] +; MOVREL: v_movreld_b32_e32 v[[ELT0:[0-9]+]] + +; IDXMODE: s_set_gpr_idx_on [[IN]], dst +; IDXMODE-NEXT: v_mov_b32_e32 v[[ELT0:[0-9]+]], v{{[0-9]+}} +; IDXMODE-NEXT: s_set_gpr_idx_off + +; GCN: buffer_store_dwordx4 v{{\[}}[[ELT0]]: +define void @insert_wo_offset(<4 x float> addrspace(1)* %out, i32 %in) { entry: %0 = insertelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, float 5.0, i32 %in - %1 = extractelement <4 x float> %0, i32 2 - store float %1, float addrspace(1)* %out + store <4 x float> %0, <4 x float> addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}insert_neg_offset_sgpr: +; GCN-LABEL: {{^}}insert_neg_offset_sgpr: ; The offset depends on the register that holds the first element of the vector. -; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} -; CHECK: v_movreld_b32_e32 v0, v{{[0-9]}} +; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} +; MOVREL: v_movreld_b32_e32 v0, 5 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} +; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst +; IDXMODE-NEXT: v_mov_b32_e32 v0, 5 +; IDXMODE-NEXT: s_set_gpr_idx_off define void @insert_neg_offset_sgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out, i32 %offset) { entry: %index = add i32 %offset, -512 @@ -153,10 +223,15 @@ entry: ; The vector indexed into is originally loaded into an SGPR rather ; than built with a reg_sequence -; CHECK-LABEL: {{^}}insert_neg_offset_sgpr_loadreg: +; GCN-LABEL: {{^}}insert_neg_offset_sgpr_loadreg: ; The offset depends on the register that holds the first element of the vector. -; CHECK: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} -; CHECK: v_movreld_b32_e32 v0, v{{[0-9]}} +; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}} +; MOVREL: v_movreld_b32_e32 v0, 5 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} +; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], dst +; IDXMODE-NEXT: v_mov_b32_e32 v0, 5 +; IDXMODE-NEXT: s_set_gpr_idx_off define void @insert_neg_offset_sgpr_loadreg(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out, <4 x i32> %vec, i32 %offset) { entry: %index = add i32 %offset, -512 @@ -165,32 +240,75 @@ entry: ret void } -; CHECK-LABEL: {{^}}insert_neg_offset_vgpr: +; GCN-LABEL: {{^}}insert_neg_offset_vgpr: ; The offset depends on the register that holds the first element of the vector. -; CHECK: v_readfirstlane_b32 -; CHECK: s_add_i32 m0, m0, 0xfffffe{{[0-9a-z]+}} -; CHECK-NEXT: v_movreld_b32_e32 v0, v{{[0-9]}} -; CHECK: s_cbranch_execnz + +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], 1{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], 2{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT2:v[0-9]+]], 3{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 4{{$}} + +; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec +; GCN: s_waitcnt lgkmcnt(0) + +; GCN: [[LOOPBB:BB[0-9]+_[0-9]+]]: +; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] + +; MOVREL: s_add_i32 m0, [[READLANE]], 0xfffffe00 +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL: v_movreld_b32_e32 [[VEC_ELT0]], 5 + +; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}} +; IDXMODE: s_set_gpr_idx_idx [[ADD_IDX]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, 5 + +; GCN: s_cbranch_execnz [[LOOPBB]] +; GCN: s_mov_b64 exec, [[SAVEEXEC]] + +; IDXMODE: s_set_gpr_idx_off + +; GCN: buffer_store_dword define void @insert_neg_offset_vgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out) { entry: %id = call i32 @llvm.amdgcn.workitem.id.x() #1 %index = add i32 %id, -512 - %value = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 5, i32 %index + %value = insertelement <4 x i32> <i32 1, i32 2, i32 3, i32 4>, i32 5, i32 %index store <4 x i32> %value, <4 x i32> addrspace(1)* %out ret void } -; CHECK-LABEL: {{^}}insert_neg_inline_offset_vgpr: +; GCN-LABEL: {{^}}insert_neg_inline_offset_vgpr: + +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], 1{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], 2{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT2:v[0-9]+]], 3{{$}} +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT3:v[0-9]+]], 4{{$}} +; GCN-DAG: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x1f4{{$}} + +; IDXMODE: s_set_gpr_idx_on 0, dst + +; GCN: s_mov_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], exec +; GCN: s_waitcnt lgkmcnt(0) + ; The offset depends on the register that holds the first element of the vector. -; CHECK: v_readfirstlane_b32 -; CHECK: s_add_i32 m0, m0, -{{[0-9]+}} -; CHECK-NEXT: v_movreld_b32_e32 v0, v{{[0-9]}} -; CHECK: s_cbranch_execnz +; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]] + +; MOVREL: s_add_i32 m0, [[READLANE]], -16 +; MOVREL: v_movreld_b32_e32 [[VEC_ELT0]], [[VAL]] + +; IDXMODE: s_add_i32 [[ADD_IDX:s[0-9]+]], [[READLANE]], -16 +; IDXMODE: s_set_gpr_idx_idx [[ADD_IDX]] +; IDXMODE: v_mov_b32_e32 [[VEC_ELT0]], [[VAL]] + +; GCN: s_cbranch_execnz + +; IDXMODE: s_set_gpr_idx_off define void @insert_neg_inline_offset_vgpr(i32 addrspace(1)* %in, <4 x i32> addrspace(1)* %out) { entry: %id = call i32 @llvm.amdgcn.workitem.id.x() #1 %index = add i32 %id, -16 - %value = insertelement <4 x i32> <i32 0, i32 1, i32 2, i32 3>, i32 5, i32 %index + %value = insertelement <4 x i32> <i32 1, i32 2, i32 3, i32 4>, i32 500, i32 %index store <4 x i32> %value, <4 x i32> addrspace(1)* %out ret void } @@ -198,41 +316,64 @@ entry: ; When the block is split to insert the loop, make sure any other ; places that need to be expanded in the same block are also handled. -; CHECK-LABEL: {{^}}extract_vgpr_offset_multiple_in_block: +; GCN-LABEL: {{^}}extract_vgpr_offset_multiple_in_block: + +; FIXME: Why is vector copied in between? + +; GCN-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]] +; GCN-DAG: s_mov_b32 [[S_ELT1:s[0-9]+]], 9 +; GCN-DAG: s_mov_b32 [[S_ELT0:s[0-9]+]], 7 +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], [[S_ELT0]] +; GCN-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], [[S_ELT1]] -; CHECK-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]] -; CHECK-DAG: s_mov_b32 [[S_ELT0:s[0-9]+]], 7 -; CHECK-DAG: s_mov_b32 [[S_ELT1:s[0-9]+]], 9 -; CHECK-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], [[S_ELT0]] -; CHECK-DAG: v_mov_b32_e32 [[VEC_ELT1:v[0-9]+]], [[S_ELT1]] -; CHECK: s_waitcnt vmcnt(0) +; IDXMODE: s_set_gpr_idx_on 0, src0 -; CHECK: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec +; GCN: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec +; GCN: s_waitcnt vmcnt(0) -; CHECK: [[LOOP0:BB[0-9]+_[0-9]+]]: -; CHECK: v_readfirstlane_b32 vcc_lo, [[IDX0]] -; CHECK: s_mov_b32 m0, vcc_lo -; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]] -; CHECK: s_and_saveexec_b64 vcc, vcc -; CHECK-NEXT: v_movrels_b32_e32 [[MOVREL0:v[0-9]+]], [[VEC_ELT0]] -; CHECK-NEXT: s_xor_b64 exec, exec, vcc -; CHECK: s_cbranch_execnz [[LOOP0]] +; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] +; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] + +; MOVREL: s_mov_b32 m0, [[READLANE]] +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL: v_movrels_b32_e32 [[MOVREL0:v[0-9]+]], [[VEC_ELT0]] + +; IDXMODE: s_set_gpr_idx_idx [[READLANE]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE: v_mov_b32_e32 [[MOVREL0:v[0-9]+]], [[VEC_ELT0]] + +; GCN-NEXT: s_xor_b64 exec, exec, vcc +; GCN-NEXT: s_cbranch_execnz [[LOOP0]] ; FIXME: Redundant copy -; CHECK: s_mov_b64 exec, [[MASK]] -; CHECK: s_mov_b64 [[MASK2:s\[[0-9]+:[0-9]+\]]], exec +; GCN: s_mov_b64 exec, [[MASK]] +; IDXMODE: s_set_gpr_idx_off + +; GCN: v_mov_b32_e32 [[VEC_ELT1_2:v[0-9]+]], [[S_ELT1]] + +; IDXMODE: s_set_gpr_idx_on 0, src0 +; GCN: s_mov_b64 [[MASK2:s\[[0-9]+:[0-9]+\]]], exec -; CHECK: [[LOOP1:BB[0-9]+_[0-9]+]]: -; CHECK: v_readfirstlane_b32 vcc_lo, [[IDX0]] -; CHECK: s_mov_b32 m0, vcc_lo -; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]] -; CHECK: s_and_saveexec_b64 vcc, vcc -; CHECK-NEXT: v_movrels_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT1]] -; CHECK-NEXT: s_xor_b64 exec, exec, vcc -; CHECK: s_cbranch_execnz [[LOOP1]] +; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] +; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] -; CHECK: buffer_store_dword [[MOVREL0]] -; CHECK: buffer_store_dword [[MOVREL1]] +; MOVREL: s_mov_b32 m0, [[READLANE]] +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL-NEXT: v_movrels_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT1_2]] + +; IDXMODE: s_set_gpr_idx_idx [[READLANE]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE-NEXT: v_mov_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT1_2]] + +; GCN-NEXT: s_xor_b64 exec, exec, vcc +; GCN: s_cbranch_execnz [[LOOP1]] + +; IDXMODE: s_set_gpr_idx_off + +; GCN: buffer_store_dword [[MOVREL0]] +; GCN: buffer_store_dword [[MOVREL1]] define void @extract_vgpr_offset_multiple_in_block(i32 addrspace(1)* %out0, i32 addrspace(1)* %out1, i32 addrspace(1)* %in) #0 { entry: %id = call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -256,41 +397,58 @@ bb2: ret void } -; CHECK-LABEL: {{^}}insert_vgpr_offset_multiple_in_block: -; CHECK-DAG: s_load_dwordx4 s{{\[}}[[S_ELT0:[0-9]+]]:[[S_ELT3:[0-9]+]]{{\]}} -; CHECK-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]] -; CHECK-DAG: v_mov_b32_e32 [[VEC_ELT0:v[0-9]+]], s[[S_ELT0]] -; CHECK-DAG: v_mov_b32 [[INS0:v[0-9]+]], 62 -; CHECK-DAG: s_waitcnt vmcnt(0) +; GCN-LABEL: {{^}}insert_vgpr_offset_multiple_in_block: +; GCN-DAG: s_load_dwordx4 s{{\[}}[[S_ELT0:[0-9]+]]:[[S_ELT3:[0-9]+]]{{\]}} +; GCN-DAG: {{buffer|flat}}_load_dword [[IDX0:v[0-9]+]] +; GCN-DAG: v_mov_b32 [[INS0:v[0-9]+]], 62 + +; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT3:[0-9]+]], s[[S_ELT3]] +; GCN: v_mov_b32_e32 v[[VEC_ELT2:[0-9]+]], s{{[0-9]+}} +; GCN: v_mov_b32_e32 v[[VEC_ELT1:[0-9]+]], s{{[0-9]+}} +; GCN: v_mov_b32_e32 v[[VEC_ELT0:[0-9]+]], s[[S_ELT0]] + +; IDXMODE: s_set_gpr_idx_on 0, dst -; CHECK: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec +; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] +; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] -; CHECK: [[LOOP0:BB[0-9]+_[0-9]+]]: -; CHECK: v_readfirstlane_b32 vcc_lo, [[IDX0]] -; CHECK: s_mov_b32 m0, vcc_lo -; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]] -; CHECK: s_and_saveexec_b64 vcc, vcc -; CHECK-NEXT: v_movreld_b32_e32 v[[MOVREL0:[0-9]+]], [[INS0]] -; CHECK-NEXT: s_xor_b64 exec, exec, vcc -; CHECK: s_cbranch_execnz [[LOOP0]] +; MOVREL: s_mov_b32 m0, [[READLANE]] +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL-NEXT: v_movreld_b32_e32 v[[VEC_ELT0]], [[INS0]] + +; IDXMODE: s_set_gpr_idx_idx [[READLANE]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE-NEXT: v_mov_b32_e32 v[[VEC_ELT0]], [[INS0]] + +; GCN-NEXT: s_xor_b64 exec, exec, vcc +; GCN: s_cbranch_execnz [[LOOP0]] ; FIXME: Redundant copy -; CHECK: s_mov_b64 exec, [[MASK]] -; CHECK: v_mov_b32_e32 [[INS1:v[0-9]+]], 63 -; CHECK: s_mov_b64 [[MASK]], exec +; GCN: s_mov_b64 exec, [[MASK:s\[[0-9]+:[0-9]+\]]] +; IDXMODE: s_set_gpr_idx_off + +; IDXMODE: s_set_gpr_idx_on 0, dst +; GCN: s_mov_b64 [[MASK]], exec + +; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]: +; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]] +; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]] + +; MOVREL: s_mov_b32 m0, [[READLANE]] +; MOVREL: s_and_saveexec_b64 vcc, vcc +; MOVREL-NEXT: v_movreld_b32_e32 v[[VEC_ELT1]], 63 + +; IDXMODE: s_set_gpr_idx_idx [[READLANE]] +; IDXMODE: s_and_saveexec_b64 vcc, vcc +; IDXMODE-NEXT: v_mov_b32_e32 v[[VEC_ELT1]], 63 -; CHECK: [[LOOP1:BB[0-9]+_[0-9]+]]: -; CHECK: v_readfirstlane_b32 vcc_lo, [[IDX0]] -; CHECK: s_mov_b32 m0, vcc_lo -; CHECK: v_cmp_eq_u32_e32 vcc, m0, [[IDX0]] -; CHECK: s_and_saveexec_b64 vcc, vcc -; CHECK-NEXT: v_movreld_b32_e32 v[[MOVREL1:[0-9]+]], [[INS1]] -; CHECK-NEXT: s_xor_b64 exec, exec, vcc -; CHECK: s_cbranch_execnz [[LOOP1]] +; GCN-NEXT: s_xor_b64 exec, exec, vcc +; GCN: s_cbranch_execnz [[LOOP1]] -; CHECK: buffer_store_dwordx4 v{{\[}}[[MOVREL0]]: +; GCN: buffer_store_dwordx4 v{{\[}}[[VEC_ELT0]]: -; CHECK: buffer_store_dword [[INS0]] +; GCN: buffer_store_dword [[INS0]] define void @insert_vgpr_offset_multiple_in_block(<4 x i32> addrspace(1)* %out0, <4 x i32> addrspace(1)* %out1, i32 addrspace(1)* %in, <4 x i32> %vec0) #0 { entry: %id = call i32 @llvm.amdgcn.workitem.id.x() #1 @@ -313,24 +471,33 @@ bb2: ret void } -; CHECK-LABEL: {{^}}extract_adjacent_blocks: -; CHECK: s_load_dword [[ARG:s[0-9]+]] -; CHECK: s_cmp_lg_i32 -; CHECK: s_cbranch_scc0 [[BB4:BB[0-9]+_[0-9]+]] +; GCN-LABEL: {{^}}extract_adjacent_blocks: +; GCN: s_load_dword [[ARG:s[0-9]+]] +; GCN: s_cmp_lg_u32 +; GCN: s_cbranch_scc0 [[BB4:BB[0-9]+_[0-9]+]] -; CHECK: buffer_load_dwordx4 -; CHECK: s_mov_b32 m0, -; CHECK: v_movrels_b32_e32 -; CHECK: s_branch [[ENDBB:BB[0-9]+_[0-9]+]] +; GCN: buffer_load_dwordx4 +; MOVREL: s_mov_b32 m0, +; MOVREL: v_movrels_b32_e32 -; CHECK: [[BB4]]: -; CHECK: buffer_load_dwordx4 -; CHECK: s_mov_b32 m0, -; CHECK: v_movrels_b32_e32 +; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, src0 +; IDXMODE: v_mov_b32_e32 +; IDXMODE: s_set_gpr_idx_off -; CHECK: [[ENDBB]]: -; CHECK: buffer_store_dword -; CHECK: s_endpgm +; GCN: s_branch [[ENDBB:BB[0-9]+_[0-9]+]] + +; GCN: [[BB4]]: +; GCN: buffer_load_dwordx4 +; MOVREL: s_mov_b32 m0, +; MOVREL: v_movrels_b32_e32 + +; IDXMODE: s_set_gpr_idx_on +; IDXMODE: v_mov_b32_e32 +; IDXMODE: s_set_gpr_idx_off + +; GCN: [[ENDBB]]: +; GCN: buffer_store_dword +; GCN: s_endpgm define void @extract_adjacent_blocks(i32 %arg) #0 { bb: %tmp = icmp eq i32 %arg, 0 @@ -339,11 +506,13 @@ bb: bb1: %tmp2 = load volatile <4 x float>, <4 x float> addrspace(1)* undef %tmp3 = extractelement <4 x float> %tmp2, i32 undef + call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp2) #0 ; Prevent block optimize out br label %bb7 bb4: %tmp5 = load volatile <4 x float>, <4 x float> addrspace(1)* undef %tmp6 = extractelement <4 x float> %tmp5, i32 undef + call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp5) #0 ; Prevent block optimize out br label %bb7 bb7: @@ -352,24 +521,33 @@ bb7: ret void } -; CHECK-LABEL: {{^}}insert_adjacent_blocks: -; CHECK: s_load_dword [[ARG:s[0-9]+]] -; CHECK: s_cmp_lg_i32 -; CHECK: s_cbranch_scc0 [[BB4:BB[0-9]+_[0-9]+]] +; GCN-LABEL: {{^}}insert_adjacent_blocks: +; GCN: s_load_dword [[ARG:s[0-9]+]] +; GCN: s_cmp_lg_u32 +; GCN: s_cbranch_scc0 [[BB4:BB[0-9]+_[0-9]+]] + +; GCN: buffer_load_dwordx4 +; MOVREL: s_mov_b32 m0, +; MOVREL: v_movreld_b32_e32 + +; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, dst +; IDXMODE: v_mov_b32_e32 +; IDXMODE: s_set_gpr_idx_off + +; GCN: s_branch [[ENDBB:BB[0-9]+_[0-9]+]] -; CHECK: buffer_load_dwordx4 -; CHECK: s_mov_b32 m0, -; CHECK: v_movreld_b32_e32 -; CHECK: s_branch [[ENDBB:BB[0-9]+_[0-9]+]] +; GCN: [[BB4]]: +; GCN: buffer_load_dwordx4 +; MOVREL: s_mov_b32 m0, +; MOVREL: v_movreld_b32_e32 -; CHECK: [[BB4]]: -; CHECK: buffer_load_dwordx4 -; CHECK: s_mov_b32 m0, -; CHECK: v_movreld_b32_e32 +; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, dst +; IDXMODE: v_mov_b32_e32 +; IDXMODE: s_set_gpr_idx_off -; CHECK: [[ENDBB]]: -; CHECK: buffer_store_dword -; CHECK: s_endpgm +; GCN: [[ENDBB]]: +; GCN: buffer_store_dword +; GCN: s_endpgm define void @insert_adjacent_blocks(i32 %arg, float %val0) #0 { bb: %tmp = icmp eq i32 %arg, 0 @@ -378,11 +556,13 @@ bb: bb1: ; preds = %bb %tmp2 = load volatile <4 x float>, <4 x float> addrspace(1)* undef %tmp3 = insertelement <4 x float> %tmp2, float %val0, i32 undef + call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp3) #0 ; Prevent block optimize out br label %bb7 bb4: ; preds = %bb %tmp5 = load volatile <4 x float>, <4 x float> addrspace(1)* undef %tmp6 = insertelement <4 x float> %tmp5, float %val0, i32 undef + call void asm sideeffect "; reg use $0", "v"(<4 x float> %tmp6) #0 ; Prevent block optimize out br label %bb7 bb7: ; preds = %bb4, %bb1 @@ -393,25 +573,48 @@ bb7: ; preds = %bb4, %bb1 ; FIXME: Should be able to fold zero input to movreld to inline imm? -; CHECK-LABEL: {{^}}multi_same_block: -; CHECK: s_load_dword [[ARG:s[0-9]+]] -; CHECK-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} -; CHECK-DAG: s_add_i32 m0, [[ARG]], -16 -; CHECK: v_movreld_b32_e32 v{{[0-9]+}}, [[ZERO]] +; GCN-LABEL: {{^}}multi_same_block: + +; GCN-DAG: v_mov_b32_e32 v[[VEC0_ELT0:[0-9]+]], 0x41880000 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41900000 +; GCN-DAG: v_mov_b32_e32 v[[VEC0_ELT2:[0-9]+]], 0x41980000 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a00000 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a80000 +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41b00000 +; GCN-DAG: s_load_dword [[ARG:s[0-9]+]] +; IDXMODE-DAG: s_add_i32 [[ARG_ADD:s[0-9]+]], [[ARG]], -16 + +; MOVREL-DAG: s_add_i32 m0, [[ARG]], -16 +; MOVREL: v_movreld_b32_e32 v[[VEC0_ELT0]], 4.0 +; GCN-NOT: m0 + +; IDXMODE: s_set_gpr_idx_on [[ARG_ADD]], dst +; IDXMODE: v_mov_b32_e32 v[[VEC0_ELT0]], 4.0 +; IDXMODE: s_set_gpr_idx_off + +; GCN: v_mov_b32_e32 v[[VEC0_ELT2]], 0x4188cccd +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x4190cccd +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x4198cccd +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a0cccd +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41a8cccd +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x41b0cccd + +; MOVREL: v_movreld_b32_e32 v[[VEC0_ELT2]], -4.0 -; CHECK: s_add_i32 m0, [[ARG]], -14 -; CHECK: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE: s_set_gpr_idx_on [[ARG_ADD]], dst +; IDXMODE: v_mov_b32_e32 v[[VEC0_ELT2]], -4.0 +; IDXMODE: s_set_gpr_idx_off -; CHECK: s_mov_b32 m0, -1 -; CHECK: ds_write_b32 -; CHECK: ds_write_b32 -; CHECK: s_endpgm +; GCN: s_mov_b32 m0, -1 +; GCN: ds_write_b32 +; GCN: ds_write_b32 +; GCN: s_endpgm define void @multi_same_block(i32 %arg) #0 { bb: %tmp1 = add i32 %arg, -16 - %tmp2 = insertelement <6 x float> <float 1.700000e+01, float 1.800000e+01, float 1.900000e+01, float 2.000000e+01, float 2.100000e+01, float 2.200000e+01>, float 0.000000e+00, i32 %tmp1 + %tmp2 = insertelement <6 x float> <float 1.700000e+01, float 1.800000e+01, float 1.900000e+01, float 2.000000e+01, float 2.100000e+01, float 2.200000e+01>, float 4.000000e+00, i32 %tmp1 %tmp3 = add i32 %arg, -16 - %tmp4 = insertelement <6 x float> <float 0x40311999A0000000, float 0x40321999A0000000, float 0x40331999A0000000, float 0x40341999A0000000, float 0x40351999A0000000, float 0x40361999A0000000>, float 0x3FB99999A0000000, i32 %tmp3 + %tmp4 = insertelement <6 x float> <float 0x40311999A0000000, float 0x40321999A0000000, float 0x40331999A0000000, float 0x40341999A0000000, float 0x40351999A0000000, float 0x40361999A0000000>, float -4.0, i32 %tmp3 %tmp5 = bitcast <6 x float> %tmp2 to <6 x i32> %tmp6 = extractelement <6 x i32> %tmp5, i32 1 %tmp7 = bitcast <6 x float> %tmp4 to <6 x i32> @@ -422,12 +625,17 @@ bb: } ; offset puts outside of superegister bounaries, so clamp to 1st element. -; CHECK-LABEL: {{^}}extract_largest_inbounds_offset: -; CHECK: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}} -; CHECK: s_load_dword [[IDX:s[0-9]+]] -; CHECK: s_mov_b32 m0, [[IDX]] -; CHECK-NEXT: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[HI_ELT]] -; CHECK: buffer_store_dword [[EXTRACT]] +; GCN-LABEL: {{^}}extract_largest_inbounds_offset: +; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}} +; GCN-DAG: s_load_dword [[IDX:s[0-9]+]] +; MOVREL: s_mov_b32 m0, [[IDX]] +; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[HI_ELT]] + +; IDXMODE: s_set_gpr_idx_on [[IDX]], src0 +; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[HI_ELT]] +; IDXMODE: s_set_gpr_idx_off + +; GCN: buffer_store_dword [[EXTRACT]] define void @extract_largest_inbounds_offset(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in, i32 %idx) { entry: %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in @@ -437,12 +645,18 @@ entry: ret void } -; CHECK-LABL: {{^}}extract_out_of_bounds_offset: -; CHECK: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}} -; CHECK: s_load_dword [[IDX:s[0-9]+]] -; CHECK: s_add_i32 m0, [[IDX]], 4 -; CHECK-NEXT: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] -; CHECK: buffer_store_dword [[EXTRACT]] +; GCN-LABEL: {{^}}extract_out_of_bounds_offset: +; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}} +; GCN-DAG: s_load_dword [[IDX:s[0-9]+]] +; MOVREL: s_add_i32 m0, [[IDX]], 4 +; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] + +; IDXMODE: s_add_i32 [[ADD_IDX:s[0-9]+]], [[IDX]], 4 +; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], src0 +; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]] +; IDXMODE: s_set_gpr_idx_off + +; GCN: buffer_store_dword [[EXTRACT]] define void @extract_out_of_bounds_offset(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in, i32 %idx) { entry: %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in @@ -459,8 +673,13 @@ entry: ; GCN: s_load_dword [[IDX_IN:s[0-9]+]] ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]] ; GCN-NOT: [[IDX_SHL]] -; GCN: s_mov_b32 m0, [[IDX_SHL]] -; GCN: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} + +; MOVREL: s_mov_b32 m0, [[IDX_SHL]] +; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} + +; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], src0 +; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE: s_set_gpr_idx_off define void @extractelement_v4i32_or_index(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in, i32 %idx.in) { entry: %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in @@ -475,8 +694,13 @@ entry: ; GCN: s_load_dword [[IDX_IN:s[0-9]+]] ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]] ; GCN-NOT: [[IDX_SHL]] -; GCN: s_mov_b32 m0, [[IDX_SHL]] -; GCN: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} + +; MOVREL: s_mov_b32 m0, [[IDX_SHL]] +; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} + +; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], dst +; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}} +; IDXMODE: s_set_gpr_idx_off define void @insertelement_v4f32_or_index(<4 x float> addrspace(1)* %out, <4 x float> %a, i32 %idx.in) nounwind { %idx.shl = shl i32 %idx.in, 2 %idx = or i32 %idx.shl, 1 @@ -485,7 +709,48 @@ define void @insertelement_v4f32_or_index(<4 x float> addrspace(1)* %out, <4 x f ret void } +; GCN-LABEL: {{^}}broken_phi_bb: +; GCN: v_mov_b32_e32 [[PHIREG:v[0-9]+]], 8 + +; GCN: s_branch [[BB2:BB[0-9]+_[0-9]+]] + +; GCN: {{^BB[0-9]+_[0-9]+}}: +; GCN: s_mov_b64 exec, +; IDXMODE: s_set_gpr_idx_off + +; GCN: [[BB2]]: +; GCN: v_cmp_le_i32_e32 vcc, s{{[0-9]+}}, [[PHIREG]] +; GCN: buffer_load_dword + +; GCN: [[REGLOOP:BB[0-9]+_[0-9]+]]: +; MOVREL: v_movreld_b32_e32 + +; IDXMODE: s_set_gpr_idx_idx +; IDXMODE: v_mov_b32_e32 +; GCN: s_cbranch_execnz [[REGLOOP]] +define void @broken_phi_bb(i32 %arg, i32 %arg1) #0 { +bb: + br label %bb2 + +bb2: ; preds = %bb4, %bb + %tmp = phi i32 [ 8, %bb ], [ %tmp7, %bb4 ] + %tmp3 = icmp slt i32 %tmp, %arg + br i1 %tmp3, label %bb4, label %bb8 + +bb4: ; preds = %bb2 + %vgpr = load volatile i32, i32 addrspace(1)* undef + %tmp5 = insertelement <8 x i32> undef, i32 undef, i32 %vgpr + %tmp6 = insertelement <8 x i32> %tmp5, i32 %arg1, i32 %vgpr + %tmp7 = extractelement <8 x i32> %tmp6, i32 0 + br label %bb2 + +bb8: ; preds = %bb2 + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #1 +declare void @llvm.amdgcn.s.barrier() #2 attributes #0 = { nounwind } attributes #1 = { nounwind readnone } +attributes #2 = { nounwind convergent } diff --git a/test/CodeGen/AMDGPU/indirect-addressing-undef.mir b/test/CodeGen/AMDGPU/indirect-addressing-undef.mir deleted file mode 100644 index 7cd35d41f30c..000000000000 --- a/test/CodeGen/AMDGPU/indirect-addressing-undef.mir +++ /dev/null @@ -1,327 +0,0 @@ -# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-lower-control-flow -o - %s | FileCheck %s -# Getting an undef that is specifically a VGPR is tricky from IR - -# CHECK-LABEL: name: extract_undef_offset_vgpr{{$}} -# CHECK: bb.1: -# CHECK: successors: %bb.2(0x40000000 / 0x80000000 = 50.00%), %bb.1(0x40000000 / 0x80000000 = 50.00%) -# CHECK: liveins: %vgpr0_vgpr1_vgpr2_vgpr3{{$}} - -# CHECK: V_READFIRSTLANE_B32 undef %vgpr10, implicit %exec -# CHECK: %vgpr0 = V_MOVRELS_B32_e32 %vgpr0, implicit %m0, implicit %exec, implicit %vgpr0_vgpr1_vgpr2_vgpr3 -# CHECK: S_CBRANCH_EXECNZ %bb.1, implicit %exec - -# CHECK: bb.2: -# CHECK: liveins: %sgpr6_sgpr7, %sgpr4_sgpr5_sgpr6_sgpr7, %sgpr4, %sgpr5, %sgpr6, %sgpr7, %sgpr4_sgpr5, %vgpr0_vgpr1_vgpr2_vgpr3, %vgpr0, %vgpr1, %vgpr2, %vgpr3, %vgpr0_vgpr1, %vgpr2_vgpr3, %vgpr0_vgpr1_vgpr2, %vgpr1_vgpr2, %vgpr1_vgpr2_vgpr3, %sgpr0_sgpr1, %sgpr0, %sgpr1{{$}} - - ---- | - target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64" - - define void @extract_undef_offset_vgpr(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { - entry: - %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in - %value = extractelement <4 x i32> %ld, i32 undef - store i32 %value, i32 addrspace(1)* %out - ret void - } - - define void @extract_undef_neg_offset_vgpr(i32 addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { - entry: - %ld = load volatile <4 x i32>, <4 x i32> addrspace(1)* %in - %value = extractelement <4 x i32> %ld, i32 undef - store i32 %value, i32 addrspace(1)* %out - ret void - } - - define void @insert_undef_offset_vgpr(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { - entry: - %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in - %value = insertelement <4 x i32> %ld, i32 5, i32 undef - store <4 x i32> %value, <4 x i32> addrspace(1)* %out - ret void - } - - define void @insert_undef_neg_offset_vgpr(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { - entry: - %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in - %value = insertelement <4 x i32> %ld, i32 5, i32 undef - store <4 x i32> %value, <4 x i32> addrspace(1)* %out - ret void - } - - define void @insert_undef_value_offset_vgpr(<4 x i32> addrspace(1)*%out, <4 x i32> addrspace(1)* %in, i32 %idx) { - entry: - %ld = load <4 x i32>, <4 x i32> addrspace(1)* %in - %value = insertelement <4 x i32> %ld, i32 undef, i32 %idx - store <4 x i32> %value, <4 x i32> addrspace(1)* %out - ret void - } - -... ---- -name: extract_undef_offset_vgpr -alignment: 0 -exposesReturnsTwice: false -hasInlineAsm: false -allVRegsAllocated: true -isSSA: false -tracksRegLiveness: true -tracksSubRegLiveness: true -liveins: - - { reg: '%sgpr0_sgpr1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0.entry: - liveins: %sgpr0_sgpr1 - - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11 - %sgpr7 = S_MOV_B32 61440 - %sgpr6 = S_MOV_B32 -1 - S_WAITCNT 127 - %vgpr0_vgpr1_vgpr2_vgpr3 = BUFFER_LOAD_DWORDX4_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9 - S_WAITCNT 3952 - %vgpr0, dead %sgpr0_sgpr1 = SI_INDIRECT_SRC_V4 killed %vgpr0_vgpr1_vgpr2_vgpr3, undef %vgpr10, 0, implicit-def dead %exec, implicit-def dead %vcc, implicit-def dead %m0, implicit %exec - S_WAITCNT 127 - BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec - S_ENDPGM - -... - -# CHECK-LABEL: name: extract_undef_neg_offset_vgpr{{$}} -# CHECK: bb.1: -# CHECK: successors: %bb.2(0x40000000 / 0x80000000 = 50.00%), %bb.1(0x40000000 / 0x80000000 = 50.00%) -# CHECK: liveins: %vgpr0_vgpr1_vgpr2_vgpr3{{$}} - -# CHECK: %vcc_lo = V_READFIRSTLANE_B32 undef %vgpr10, implicit %exec -# CHECK: %m0 = S_MOV_B32 %vcc_lo -# CHECK: %m0 = S_ADD_I32 %m0, -7, implicit-def %scc -# CHECK: %vgpr0 = V_MOVRELS_B32_e32 %vgpr0, implicit %m0, implicit %exec, implicit %vgpr0_vgpr1_vgpr2_vgpr3 -# CHECK: S_CBRANCH_EXECNZ %bb.1, implicit %exec - -# CHECK: bb.2: -# CHECK: liveins: %sgpr6_sgpr7, %sgpr4_sgpr5_sgpr6_sgpr7, %sgpr4, %sgpr5, %sgpr6, %sgpr7, %sgpr4_sgpr5, %vgpr0_vgpr1_vgpr2_vgpr3, %vgpr0, %vgpr1, %vgpr2, %vgpr3, %vgpr0_vgpr1, %vgpr2_vgpr3, %vgpr0_vgpr1_vgpr2, %vgpr1_vgpr2, %vgpr1_vgpr2_vgpr3, %sgpr0_sgpr1, %sgpr0, %sgpr1 - -name: extract_undef_neg_offset_vgpr -alignment: 0 -exposesReturnsTwice: false -hasInlineAsm: false -allVRegsAllocated: true -isSSA: false -tracksRegLiveness: true -tracksSubRegLiveness: true -liveins: - - { reg: '%sgpr0_sgpr1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0.entry: - liveins: %sgpr0_sgpr1 - - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11 - %sgpr7 = S_MOV_B32 61440 - %sgpr6 = S_MOV_B32 -1 - S_WAITCNT 127 - %vgpr0_vgpr1_vgpr2_vgpr3 = BUFFER_LOAD_DWORDX4_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9 - S_WAITCNT 3952 - %vgpr0, dead %sgpr0_sgpr1 = SI_INDIRECT_SRC_V4 killed %vgpr0_vgpr1_vgpr2_vgpr3, undef %vgpr10, -7, implicit-def dead %exec, implicit-def dead %vcc, implicit-def dead %m0, implicit %exec - S_WAITCNT 127 - BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec - S_ENDPGM - -... - -# CHECK-LABEL: name: insert_undef_offset_vgpr{{$}} -# CHECK: bb.1: -# CHECK: successors: %bb.2(0x40000000 / 0x80000000 = 50.00%), %bb.1(0x40000000 / 0x80000000 = 50.00%) -# CHECK: liveins: %vgpr4, %vgpr0_vgpr1_vgpr2_vgpr3{{$}} - -# CHECK: %vcc_lo = V_READFIRSTLANE_B32 undef %vgpr10, implicit %exec -# CHECK: %m0 = S_MOV_B32 %vcc_lo -# CHECK: %vgpr0 = V_MOVRELD_B32_e32 %vgpr4, implicit %m0, implicit %exec, implicit %vgpr0_vgpr1_vgpr2_vgpr3 -# CHECK: S_CBRANCH_EXECNZ %bb.1, implicit %exec - -# CHECK: bb.2: -# CHECK: liveins: %sgpr6_sgpr7, %sgpr7, %sgpr4_sgpr5, %sgpr5, %sgpr4_sgpr5_sgpr6_sgpr7, %sgpr6, %sgpr4, %vgpr0_vgpr1_vgpr2_vgpr3, %vgpr0, %vgpr1, %vgpr2, %vgpr3, %vgpr0_vgpr1, %vgpr2_vgpr3, %vgpr0_vgpr1_vgpr2, %vgpr1_vgpr2, %vgpr1_vgpr2_vgpr3, %vgpr4, %sgpr0_sgpr1, %sgpr0, %sgpr1 - -name: insert_undef_offset_vgpr -alignment: 0 -exposesReturnsTwice: false -hasInlineAsm: false -allVRegsAllocated: true -isSSA: false -tracksRegLiveness: true -tracksSubRegLiveness: true -liveins: - - { reg: '%sgpr0_sgpr1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0.entry: - liveins: %sgpr0_sgpr1 - - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - %sgpr7 = S_MOV_B32 61440 - %sgpr6 = S_MOV_B32 -1 - %vgpr4 = V_MOV_B32_e32 5, implicit %exec - S_WAITCNT 127 - %vgpr0_vgpr1_vgpr2_vgpr3 = BUFFER_LOAD_DWORDX4_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (load 16 from %ir.in) - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - S_WAITCNT 3952 - %vgpr0_vgpr1_vgpr2_vgpr3, dead %sgpr0_sgpr1 = SI_INDIRECT_DST_V4 %vgpr0_vgpr1_vgpr2_vgpr3, undef %vgpr10, 0, killed %vgpr4, implicit-def dead %exec, implicit-def dead %vcc, implicit-def dead %m0, implicit %exec - S_WAITCNT 127 - BUFFER_STORE_DWORDX4_OFFSET killed %vgpr0_vgpr1_vgpr2_vgpr3, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (store 16 into %ir.out) - S_ENDPGM - -... - -# CHECK-LABEL: name: insert_undef_neg_offset_vgpr{{$}} -# CHECK: bb.1: -# CHECK: successors: %bb.2(0x40000000 / 0x80000000 = 50.00%), %bb.1(0x40000000 / 0x80000000 = 50.00%) -# CHECK: liveins: %vgpr4, %vgpr0_vgpr1_vgpr2_vgpr3{{$}} - -# CHECK: %vcc_lo = V_READFIRSTLANE_B32 undef %vgpr10, implicit %exec -# CHECK: %m0 = S_MOV_B32 %vcc_lo -# CHECK: %m0 = S_ADD_I32 %m0, -7, implicit-def %scc -# CHECK: %vgpr0 = V_MOVRELD_B32_e32 %vgpr4, implicit %m0, implicit %exec, implicit %vgpr0_vgpr1_vgpr2_vgpr3 -# CHECK: S_CBRANCH_EXECNZ %bb.1, implicit %exec - -# CHECK: bb.2: -# CHECK: liveins: %sgpr6_sgpr7, %sgpr7, %sgpr4_sgpr5, %sgpr5, %sgpr4_sgpr5_sgpr6_sgpr7, %sgpr6, %sgpr4, %vgpr0_vgpr1_vgpr2_vgpr3, %vgpr0, %vgpr1, %vgpr2, %vgpr3, %vgpr0_vgpr1, %vgpr2_vgpr3, %vgpr0_vgpr1_vgpr2, %vgpr1_vgpr2, %vgpr1_vgpr2_vgpr3, %vgpr4, %sgpr0_sgpr1, %sgpr0, %sgpr1{{$}} - -name: insert_undef_neg_offset_vgpr -alignment: 0 -exposesReturnsTwice: false -hasInlineAsm: false -allVRegsAllocated: true -isSSA: false -tracksRegLiveness: true -tracksSubRegLiveness: true -liveins: - - { reg: '%sgpr0_sgpr1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0.entry: - liveins: %sgpr0_sgpr1 - - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - %sgpr7 = S_MOV_B32 61440 - %sgpr6 = S_MOV_B32 -1 - %vgpr4 = V_MOV_B32_e32 5, implicit %exec - S_WAITCNT 127 - %vgpr0_vgpr1_vgpr2_vgpr3 = BUFFER_LOAD_DWORDX4_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (load 16 from %ir.in) - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - S_WAITCNT 3952 - %vgpr0_vgpr1_vgpr2_vgpr3, dead %sgpr0_sgpr1 = SI_INDIRECT_DST_V4 %vgpr0_vgpr1_vgpr2_vgpr3, undef %vgpr10, -7, killed %vgpr4, implicit-def dead %exec, implicit-def dead %vcc, implicit-def dead %m0, implicit %exec - S_WAITCNT 127 - BUFFER_STORE_DWORDX4_OFFSET killed %vgpr0_vgpr1_vgpr2_vgpr3, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (store 16 into %ir.out) - S_ENDPGM - -... - -# CHECK-LABEL: insert_undef_value_offset_vgpr{{$}} -# CHECK: bb.1: -# CHECK: successors: %bb.2(0x40000000 / 0x80000000 = 50.00%), %bb.1(0x40000000 / 0x80000000 = 50.00%) -# CHECK: liveins: %vgpr4, %vgpr0_vgpr1_vgpr2_vgpr3{{$}} - -# CHECK: %vcc_lo = V_READFIRSTLANE_B32 %vgpr4, implicit %exec -# CHECK: %m0 = S_MOV_B32 %vcc_lo -# CHECK: %vgpr0 = V_MOVRELD_B32_e32 undef %vgpr10, implicit %m0, implicit %exec, implicit %vgpr0_vgpr1_vgpr2_vgpr3 -# CHECK: S_CBRANCH_EXECNZ %bb.1, implicit %exec - -# CHECK: bb.2: -# CHECK: liveins: %sgpr6_sgpr7, %sgpr7, %sgpr4_sgpr5, %sgpr5, %sgpr4_sgpr5_sgpr6_sgpr7, %sgpr6, %sgpr4, %vgpr0_vgpr1_vgpr2_vgpr3, %vgpr0, %vgpr1, %vgpr2, %vgpr3, %vgpr0_vgpr1, %vgpr2_vgpr3, %vgpr0_vgpr1_vgpr2, %vgpr1_vgpr2, %vgpr1_vgpr2_vgpr3, %vgpr4, %sgpr0_sgpr1, %sgpr0, %sgpr1{{$}} - -name: insert_undef_value_offset_vgpr -alignment: 0 -exposesReturnsTwice: false -hasInlineAsm: false -allVRegsAllocated: true -isSSA: false -tracksRegLiveness: true -tracksSubRegLiveness: true -liveins: - - { reg: '%sgpr0_sgpr1' } -frameInfo: - isFrameAddressTaken: false - isReturnAddressTaken: false - hasStackMap: false - hasPatchPoint: false - stackSize: 0 - offsetAdjustment: 0 - maxAlignment: 0 - adjustsStack: false - hasCalls: false - maxCallFrameSize: 0 - hasOpaqueSPAdjustment: false - hasVAStart: false - hasMustTailInVarArgFunc: false -body: | - bb.0.entry: - liveins: %sgpr0_sgpr1 - - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM %sgpr0_sgpr1, 11 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - %sgpr7 = S_MOV_B32 61440 - %sgpr6 = S_MOV_B32 -1 - %vgpr4 = V_MOV_B32_e32 2, implicit %exec - S_WAITCNT 127 - %vgpr0_vgpr1_vgpr2_vgpr3 = BUFFER_LOAD_DWORDX4_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (load 16 from %ir.in) - %sgpr4_sgpr5 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 9 :: (non-temporal invariant load 8 from `i64 addrspace(2)* undef`) - S_WAITCNT 3952 - %vgpr0_vgpr1_vgpr2_vgpr3, dead %sgpr0_sgpr1 = SI_INDIRECT_DST_V4 %vgpr0_vgpr1_vgpr2_vgpr3, killed %vgpr4, 0, undef %vgpr10, implicit-def dead %exec, implicit-def dead %vcc, implicit-def dead %m0, implicit %exec - S_WAITCNT 127 - BUFFER_STORE_DWORDX4_OFFSET killed %vgpr0_vgpr1_vgpr2_vgpr3, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (store 16 into %ir.out) - S_ENDPGM - -... diff --git a/test/CodeGen/AMDGPU/indirect-private-64.ll b/test/CodeGen/AMDGPU/indirect-private-64.ll index 1f851f9de535..4c9ef2e61f83 100644 --- a/test/CodeGen/AMDGPU/indirect-private-64.ll +++ b/test/CodeGen/AMDGPU/indirect-private-64.ll @@ -121,4 +121,4 @@ define void @private_access_v2i64_alloca(<2 x i64> addrspace(1)* noalias %out, < } attributes #0 = { convergent nounwind } -attributes #1 = { nounwind "amdgpu-max-waves-per-eu"="2" "amdgpu-max-work-group-size"="64" } +attributes #1 = { nounwind "amdgpu-waves-per-eu"="1,2" "amdgpu-flat-work-group-size"="64,64" } diff --git a/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll b/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll index f6e39b3d8306..990f33518ab9 100644 --- a/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll +++ b/test/CodeGen/AMDGPU/infinite-loop-evergreen.ll @@ -1,4 +1,5 @@ ; XFAIL: * +; REQUIRES: asserts ; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck %s define void @inf_loop_irreducible_cfg() nounwind { diff --git a/test/CodeGen/AMDGPU/inline-asm.ll b/test/CodeGen/AMDGPU/inline-asm.ll index 1f5b8be15e2e..cef1b114cd32 100644 --- a/test/CodeGen/AMDGPU/inline-asm.ll +++ b/test/CodeGen/AMDGPU/inline-asm.ll @@ -40,12 +40,12 @@ endif: ; CHECK-LABEL: {{^}}v_cmp_asm: ; CHECK: v_mov_b32_e32 [[SRC:v[0-9]+]], s{{[0-9]+}} -; CHECK: v_cmp_ne_i32_e64 s{{\[}}[[MASK_LO:[0-9]+]]:[[MASK_HI:[0-9]+]]{{\]}}, 0, [[SRC]] +; CHECK: v_cmp_ne_u32_e64 s{{\[}}[[MASK_LO:[0-9]+]]:[[MASK_HI:[0-9]+]]{{\]}}, 0, [[SRC]] ; CHECK-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], s[[MASK_LO]] ; CHECK-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], s[[MASK_HI]] ; CHECK: buffer_store_dwordx2 v{{\[}}[[V_LO]]:[[V_HI]]{{\]}} define void @v_cmp_asm(i64 addrspace(1)* %out, i32 %in) { - %sgpr = tail call i64 asm "v_cmp_ne_i32_e64 $0, 0, $1", "=s,v"(i32 %in) + %sgpr = tail call i64 asm "v_cmp_ne_u32_e64 $0, 0, $1", "=s,v"(i32 %in) store i64 %sgpr, i64 addrspace(1)* %out ret void } diff --git a/test/CodeGen/AMDGPU/inline-calls.ll b/test/CodeGen/AMDGPU/inline-calls.ll index 33a4c832e75e..4541a902c1b8 100644 --- a/test/CodeGen/AMDGPU/inline-calls.ll +++ b/test/CodeGen/AMDGPU/inline-calls.ll @@ -10,6 +10,7 @@ entry: } ; CHECK: {{^}}kernel: +; CHECK-NOT: call define void @kernel(i32 addrspace(1)* %out) { entry: %tmp0 = call i32 @func(i32 1) @@ -18,8 +19,32 @@ entry: } ; CHECK: {{^}}kernel2: +; CHECK-NOT: call define void @kernel2(i32 addrspace(1)* %out) { entry: call void @kernel(i32 addrspace(1)* %out) ret void } + +; CHECK-NOT: func_alias +@func_alias = alias i32 (i32), i32 (i32)* @func + +; CHECK: {{^}}kernel3: +; CHECK-NOT: call +define void @kernel3(i32 addrspace(1)* %out) { +entry: + %tmp0 = call i32 @func_alias(i32 1) + store i32 %tmp0, i32 addrspace(1)* %out + ret void +} + +; CHECK-NOT: kernel_alias +@kernel_alias = alias void (i32 addrspace(1)*), void (i32 addrspace(1)*)* @kernel + +; CHECK: {{^}}kernel4: +; CHECK-NOT: call +define void @kernel4(i32 addrspace(1)* %out) { +entry: + call void @kernel_alias(i32 addrspace(1)* %out) + ret void +} diff --git a/test/CodeGen/AMDGPU/inline-constraints.ll b/test/CodeGen/AMDGPU/inline-constraints.ll index 78868710c6a2..1bcbd14009ce 100644 --- a/test/CodeGen/AMDGPU/inline-constraints.ll +++ b/test/CodeGen/AMDGPU/inline-constraints.ll @@ -21,3 +21,49 @@ entry: %s256 = tail call <8 x i32> asm sideeffect "s_load_dwordx8 $0, $1", "=s,s"(i32 addrspace(1)* %ptr) ret void } + +; FIXME: Should be able to avoid copy +; GCN-LABEL: {{^}}inline_sreg_constraint_m0: +; GCN: s_mov_b32 m0, -1 +; GCN: s_mov_b32 [[COPY_M0:s[0-9]+]], m0 +; GCN: ; use [[COPY_M0]] +define void @inline_sreg_constraint_m0() { + %m0 = tail call i32 asm sideeffect "s_mov_b32 m0, -1", "={M0}"() + tail call void asm sideeffect "; use $0", "s"(i32 %m0) + ret void +} + +; GCN-LABEL: {{^}}inline_sreg_constraint_imm_i32: +; GCN: s_mov_b32 [[REG:s[0-9]+]], 32 +; GCN: ; use [[REG]] +define void @inline_sreg_constraint_imm_i32() { + tail call void asm sideeffect "; use $0", "s"(i32 32) + ret void +} + +; GCN-LABEL: {{^}}inline_sreg_constraint_imm_f32: +; GCN: s_mov_b32 [[REG:s[0-9]+]], 1.0 +; GCN: ; use [[REG]] +define void @inline_sreg_constraint_imm_f32() { + tail call void asm sideeffect "; use $0", "s"(float 1.0) + ret void +} + +; FIXME: Should be able to use s_mov_b64 +; GCN-LABEL: {{^}}inline_sreg_constraint_imm_i64: +; GCN-DAG: s_mov_b32 s[[REG_LO:[0-9]+]], -4{{$}} +; GCN-DAG: s_mov_b32 s[[REG_HI:[0-9]+]], -1{{$}} +; GCN: ; use s{{\[}}[[REG_LO]]:[[REG_HI]]{{\]}} +define void @inline_sreg_constraint_imm_i64() { + tail call void asm sideeffect "; use $0", "s"(i64 -4) + ret void +} + +; GCN-LABEL: {{^}}inline_sreg_constraint_imm_f64: +; GCN-DAG: s_mov_b32 s[[REG_LO:[0-9]+]], 0{{$}} +; GCN-DAG: s_mov_b32 s[[REG_HI:[0-9]+]], 0x3ff00000{{$}} +; GCN: ; use s{{\[}}[[REG_LO]]:[[REG_HI]]{{\]}} +define void @inline_sreg_constraint_imm_f64() { + tail call void asm sideeffect "; use $0", "s"(double 1.0) + ret void +} diff --git a/test/CodeGen/AMDGPU/inlineasm-16.ll b/test/CodeGen/AMDGPU/inlineasm-16.ll new file mode 100644 index 000000000000..75f3158937dc --- /dev/null +++ b/test/CodeGen/AMDGPU/inlineasm-16.ll @@ -0,0 +1,41 @@ +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: not llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=SICI %s +; RUN: not llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=SICI %s + +; GCN-LABEL: {{^}}s_input_output_i16: +; SICI: error: couldn't allocate output register for constraint 's' +; SICI: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_i16() #0 { + %v = tail call i16 asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(i16 %v) #0 + ret void +} + +; GCN-LABEL: {{^}}v_input_output_i16: +; SICI: error: couldn't allocate output register for constraint 'v' +; SICI: error: couldn't allocate input reg for constraint 'v' +define void @v_input_output_i16() #0 { + %v = tail call i16 asm sideeffect "v_mov_b32 $0, -1", "=v"() #0 + tail call void asm sideeffect "; use $0", "v"(i16 %v) + ret void +} + +; GCN-LABEL: {{^}}s_input_output_f16: +; SICI: error: couldn't allocate output register for constraint 's' +; SICI: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_f16() #0 { + %v = tail call half asm sideeffect "s_mov_b32 $0, -1", "=s"() #0 + tail call void asm sideeffect "; use $0", "s"(half %v) + ret void +} + +; GCN-LABEL: {{^}}v_input_output_f16: +; SICI: error: couldn't allocate output register for constraint 'v' +; SICI: error: couldn't allocate input reg for constraint 'v' +define void @v_input_output_f16() #0 { + %v = tail call half asm sideeffect "v_mov_b32 $0, -1", "=v"() #0 + tail call void asm sideeffect "; use $0", "v"(half %v) + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll b/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll new file mode 100644 index 000000000000..2eb21f07e0ec --- /dev/null +++ b/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll @@ -0,0 +1,83 @@ +; RUN: not llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GCN -check-prefix=CI %s +; RUN: not llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN: error: couldn't allocate output register for constraint 's' +; GCN: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_i8() { + %v = tail call i8 asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(i8 %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 'v' +; GCN: error: couldn't allocate input reg for constraint 'v' +define void @v_input_output_i8() { + %v = tail call i8 asm sideeffect "v_mov_b32 $0, -1", "=v"() + tail call void asm sideeffect "; use $0", "v"(i8 %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 's' +; GCN: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_i128() { + %v = tail call i128 asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(i128 %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 's' +; GCN: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_v8f16() { + %v = tail call <8 x half> asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(<8 x half> %v) + ret void +} + +; CI: error: couldn't allocate output register for constraint 's' +; CI: error: couldn't allocate input reg for constraint 's' +; VI-NOT: error +define void @s_input_output_f16() { + %v = tail call half asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(half %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 's' +; GCN: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_v2f16() { + %v = tail call <2 x half> asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(<2 x half> %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 'v' +; GCN: error: couldn't allocate input reg for constraint 'v' +define void @v_input_output_v2f16() { + %v = tail call <2 x half> asm sideeffect "v_mov_b32 $0, -1", "=v"() + tail call void asm sideeffect "; use $0", "v"(<2 x half> %v) + ret void +} + +; CI: error: couldn't allocate output register for constraint 's' +; CI: error: couldn't allocate input reg for constraint 's' +; VI-NOT: error +define void @s_input_output_i16() { + %v = tail call i16 asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(i16 %v) + ret void +} + +; GCN: error: couldn't allocate output register for constraint 's' +; GCN: error: couldn't allocate input reg for constraint 's' +define void @s_input_output_v2i16() { + %v = tail call <2 x i16> asm sideeffect "s_mov_b32 $0, -1", "=s"() + tail call void asm sideeffect "; use $0", "s"(<2 x i16> %v) + ret void +} + +; FIXME: Crash in codegen prepare +; define void @s_input_output_i3() { +; %v = tail call i3 asm sideeffect "s_mov_b32 $0, -1", "=s"() +; tail call void asm sideeffect "; use $0", "s"(i3 %v) +; ret void +; } diff --git a/test/CodeGen/AMDGPU/insert-waits-exp.mir b/test/CodeGen/AMDGPU/insert-waits-exp.mir new file mode 100644 index 000000000000..9aaa374ed28e --- /dev/null +++ b/test/CodeGen/AMDGPU/insert-waits-exp.mir @@ -0,0 +1,63 @@ +# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-insert-waits -o - %s | FileCheck %s +--- | + define amdgpu_ps <4 x float> @exp_done_waitcnt(<4 x i32> inreg, <4 x i32> inreg, i32 inreg %w, float %v) { + %a = load volatile float, float addrspace(1)* undef + %b = load volatile float, float addrspace(1)* undef + %c = load volatile float, float addrspace(1)* undef + %d = load volatile float, float addrspace(1)* undef + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %a, float %b, float %c, float %d) + ret <4 x float> <float 5.000000e-01, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00> + } + + declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) + + attributes #0 = { readnone } + attributes #1 = { nounwind } + +... +--- + +# CHECK-LABEL: name: exp_done_waitcnt{{$}} +# CHECK: EXP_DONE +# CHECK-NEXT: S_WAITCNT 3855 +# CHECK: %vgpr0 = V_MOV_B32 +# CHECK: %vgpr1 = V_MOV_B32 +# CHECK: %vgpr2 = V_MOV_B32 +# CHECK: %vgpr3 = V_MOV_B32 +name: exp_done_waitcnt +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0 (%ir-block.2): + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `float addrspace(1)* undef`) + %vgpr1 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `float addrspace(1)* undef`) + %vgpr2 = BUFFER_LOAD_DWORD_OFFSET %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `float addrspace(1)* undef`) + %vgpr3 = BUFFER_LOAD_DWORD_OFFSET killed %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `float addrspace(1)* undef`) + EXP_DONE 0, killed %vgpr0, killed %vgpr1, killed %vgpr2, killed %vgpr3, -1, -1, 15, implicit %exec + %vgpr0 = V_MOV_B32_e32 1056964608, implicit %exec + %vgpr1 = V_MOV_B32_e32 1065353216, implicit %exec + %vgpr2 = V_MOV_B32_e32 1073741824, implicit %exec + %vgpr3 = V_MOV_B32_e32 1082130432, implicit %exec + SI_RETURN killed %vgpr0, killed %vgpr1, killed %vgpr2, killed %vgpr3 + +... diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.ll b/test/CodeGen/AMDGPU/insert_vector_elt.ll index 367e7f734556..7351665f06e4 100644 --- a/test/CodeGen/AMDGPU/insert_vector_elt.ll +++ b/test/CodeGen/AMDGPU/insert_vector_elt.ll @@ -15,7 +15,7 @@ ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} -; GCN-DAG: v_mov_b32_e32 [[CONSTREG:v[0-9]+]], 0x40a00000 +; GCN-DAG: s_mov_b32 [[CONSTREG:s[0-9]+]], 0x40a00000 ; GCN-DAG: v_mov_b32_e32 v[[LOW_REG:[0-9]+]], [[CONSTREG]] ; GCN: buffer_store_dwordx4 v{{\[}}[[LOW_REG]]: define void @insertelement_v4f32_0(<4 x float> addrspace(1)* %out, <4 x float> %a) nounwind { @@ -145,8 +145,7 @@ define void @dynamic_insertelement_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> } ; GCN-LABEL: {{^}}dynamic_insertelement_v3i32: -; GCN: v_mov_b32_e32 [[CONST:v[0-9]+]], 5 -; GCN: v_movreld_b32_e32 v[[LOW_RESULT_REG:[0-9]+]], [[CONST]] +; GCN: v_movreld_b32_e32 v[[LOW_RESULT_REG:[0-9]+]], 5 ; GCN-DAG: buffer_store_dwordx2 {{v\[}}[[LOW_RESULT_REG]]: ; GCN-DAG: buffer_store_dword v define void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, i32 %b) nounwind { @@ -156,10 +155,12 @@ define void @dynamic_insertelement_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> } ; GCN-LABEL: {{^}}dynamic_insertelement_v4i32: -; GCN: v_movreld_b32 +; GCN: s_load_dword [[SVAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x12|0x48}} +; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]] +; GCN: v_movreld_b32_e32 v{{[0-9]+}}, [[VVAL]] ; GCN: buffer_store_dwordx4 -define void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, i32 %b) nounwind { - %vecins = insertelement <4 x i32> %a, i32 5, i32 %b +define void @dynamic_insertelement_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, i32 %b, i32 %val) nounwind { + %vecins = insertelement <4 x i32> %a, i32 %val, i32 %b store <4 x i32> %vecins, <4 x i32> addrspace(1)* %out, align 16 ret void } @@ -206,12 +207,14 @@ define void @dynamic_insertelement_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> ; GCN: buffer_load_ushort v{{[0-9]+}}, off ; GCN: buffer_load_ushort v{{[0-9]+}}, off -; GCN-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:6 -; GCN-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:4 -; GCN-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:2 -; GCN-DAG: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:6 +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4 +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2 +; GCN-DAG: buffer_store_short v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN: buffer_store_short v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN: s_waitcnt + ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort ; GCN: buffer_load_ushort @@ -228,8 +231,8 @@ define void @dynamic_insertelement_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> ; GCN: buffer_load_ubyte v{{[0-9]+}}, off ; GCN: buffer_load_ubyte v{{[0-9]+}}, off -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:1 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:1 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} @@ -248,9 +251,9 @@ define void @dynamic_insertelement_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> %a ; GCN: buffer_load_ubyte v{{[0-9]+}}, off ; GCN: buffer_load_ubyte v{{[0-9]+}}, off -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:2 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:1 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:1 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} @@ -272,10 +275,10 @@ define void @dynamic_insertelement_v3i8(<3 x i8> addrspace(1)* %out, <3 x i8> %a ; GCN: buffer_load_ubyte v{{[0-9]+}}, off ; GCN: buffer_load_ubyte v{{[0-9]+}}, off -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:3 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:2 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen offset:1 -; GCN-DAG: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:3 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:2 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:1 +; GCN-DAG: buffer_store_byte v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}}{{$}} ; GCN: buffer_store_byte v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen{{$}} @@ -332,25 +335,25 @@ endif: } ; GCN-LABEL: {{^}}dynamic_insertelement_v2f64: -; GCN: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x11|0x44}}{{$}} +; GCN-DAG: s_load_dwordx4 s{{\[}}[[A_ELT0:[0-9]+]]:[[A_ELT3:[0-9]+]]{{\]}} +; GCN-DAG: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0x11|0x44}}{{$}} + ; GCN-DAG: s_lshl_b32 [[SCALEDIDX:s[0-9]+]], [[IDX]], 1{{$}} -; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 0{{$}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} +; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 0x40200000 -; GCN: s_mov_b32 m0, [[SCALEDIDX]] -; GCN: v_movreld_b32_e32 v{{[0-9]+}}, [[ELT0]] +; GCN-DAG: s_mov_b32 m0, [[SCALEDIDX]] +; GCN: v_movreld_b32_e32 v{{[0-9]+}}, 0 -; Increment to next element. -; FIXME: Should be able to manipulate m0 directly instead of add and -; copy. +; Increment to next element folded into base register, but FileCheck +; can't do math expressions + +; FIXME: Should be able to manipulate m0 directly instead of s_lshl_b32 + copy to m0 -; FIXME: Should avoid resetting m0 to same value -; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 0x40200000 -; GCN-DAG: s_mov_b32 m0, [[SCALEDIDX]] ; GCN: v_movreld_b32_e32 v{{[0-9]+}}, [[ELT1]] ; GCN: buffer_store_dwordx4 @@ -361,14 +364,10 @@ define void @dynamic_insertelement_v2f64(<2 x double> addrspace(1)* %out, <2 x d ret void } -; FIXME: Inline immediate should be folded into v_movreld_b32. ; GCN-LABEL: {{^}}dynamic_insertelement_v2i64: -; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 5{{$}} -; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 0{{$}} - -; GCN-DAG: v_movreld_b32_e32 v{{[0-9]+}}, [[ELT0]] -; GCN-DAG: v_movreld_b32_e32 v{{[0-9]+}}, [[ELT1]] +; GCN-DAG: v_movreld_b32_e32 v{{[0-9]+}}, 5 +; GCN-DAG: v_movreld_b32_e32 v{{[0-9]+}}, 0 ; GCN: buffer_store_dwordx4 ; GCN: s_endpgm @@ -393,15 +392,15 @@ define void @dynamic_insertelement_v3i64(<3 x i64> addrspace(1)* %out, <3 x i64> ; Stack store -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:16{{$}} +; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}}{{$}} +; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:16{{$}} ; Write element ; GCN: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} ; Stack reload -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:16{{$}} -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:16{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}}{{$}} ; Store result ; GCN: buffer_store_dwordx4 @@ -416,19 +415,19 @@ define void @dynamic_insertelement_v4f64(<4 x double> addrspace(1)* %out, <4 x d } ; GCN-LABEL: {{^}}dynamic_insertelement_v8f64: -; GCN: SCRATCH_RSRC_DWORD +; GCN-DAG: SCRATCH_RSRC_DWORD -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:16{{$}} -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:32{{$}} -; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:48{{$}} +; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:16{{$}} +; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:32{{$}} +; GCN-DAG: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:48{{$}} +; GCN: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}}{{$}} ; GCN: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:16{{$}} -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen offset:16{{$}} -; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:48{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:32{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:16{{$}} +; GCN-DAG: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}}{{$}} ; GCN: buffer_store_dwordx4 ; GCN: buffer_store_dwordx4 diff --git a/test/CodeGen/AMDGPU/inserted-wait-states.mir b/test/CodeGen/AMDGPU/inserted-wait-states.mir new file mode 100644 index 000000000000..7cc9c7c1d923 --- /dev/null +++ b/test/CodeGen/AMDGPU/inserted-wait-states.mir @@ -0,0 +1,333 @@ +# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass post-RA-hazard-rec %s -o - | FileCheck %s -check-prefixes=GCN +# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass post-RA-hazard-rec %s -o - | FileCheck %s -check-prefixes=GCN,CIVI +# RUN: llc -march=amdgcn -mcpu=fiji -run-pass post-RA-hazard-rec %s -o - | FileCheck %s -check-prefixes=GCN,CIVI,VI + +--- | + define void @div_fmas() { ret void } + define void @s_getreg() { ret void } + define void @s_setreg() { ret void } + define void @vmem_gt_8dw_store() { ret void } + define void @readwrite_lane() { ret void } + define void @rfe() { ret void } +... +--- +# GCN-LABEL: name: div_fmas + +# GCN-LABEL: bb.0: +# GCN: S_MOV_B64 +# GCN-NOT: S_NOP +# GCN: V_DIV_FMAS + +# GCN-LABEL: bb.1: +# GCN: V_CMP_EQ_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_DIV_FMAS_F32 + +# GCN-LABEL: bb.2: +# GCN: V_CMP_EQ_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_DIV_FMAS_F32 + +# GCN-LABEL: bb.3: +# GCN: V_DIV_SCALE_F32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_DIV_FMAS_F32 +name: div_fmas + +body: | + bb.0: + successors: %bb.1 + %vcc = S_MOV_B64 0 + %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + implicit %vcc = V_CMP_EQ_I32_e32 %vgpr1, %vgpr2, implicit %exec + %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3 + %vcc = V_CMP_EQ_I32_e64 %vgpr1, %vgpr2, implicit %exec + %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec + S_BRANCH %bb.3 + + bb.3: + %vgpr4, %vcc = V_DIV_SCALE_F32 0, %vgpr1, 0, %vgpr1, 0, %vgpr3, 0, 0, implicit %exec + %vgpr0 = V_DIV_FMAS_F32 0, %vgpr1, 0, %vgpr2, 0, %vgpr3, 0, 0, implicit %vcc, implicit %exec + S_ENDPGM + +... + +... +--- +# GCN-LABEL: name: s_getreg + +# GCN-LABEL: bb.0: +# GCN: S_SETREG +# GCN: S_NOP 0 +# GCN: S_NOP 0 +# GCN: S_GETREG + +# GCN-LABEL: bb.1: +# GCN: S_SETREG_IMM32 +# GCN: S_NOP 0 +# GCN: S_NOP 0 +# GCN: S_GETREG + +# GCN-LABEL: bb.2: +# GCN: S_SETREG +# GCN: S_NOP 0 +# GCN: S_GETREG + +# GCN-LABEL: bb.3: +# GCN: S_SETREG +# GCN-NEXT: S_GETREG + +name: s_getreg + +body: | + bb.0: + successors: %bb.1 + S_SETREG_B32 %sgpr0, 1 + %sgpr1 = S_GETREG_B32 1 + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + S_SETREG_IMM32_B32 0, 1 + %sgpr1 = S_GETREG_B32 1 + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3 + S_SETREG_B32 %sgpr0, 1 + %sgpr1 = S_MOV_B32 0 + %sgpr2 = S_GETREG_B32 1 + S_BRANCH %bb.3 + + bb.3: + S_SETREG_B32 %sgpr0, 0 + %sgpr1 = S_GETREG_B32 1 + S_ENDPGM +... + +... +--- +# GCN-LABEL: name: s_setreg + +# GCN-LABEL: bb.0: +# GCN: S_SETREG +# GCN: S_NOP 0 +# VI: S_NOP 0 +# GCN-NEXT: S_SETREG + +# GCN-LABEL: bb.1: +# GCN: S_SETREG +# GCN: S_NOP 0 +# VI: S_NOP 0 +# GCN-NEXT: S_SETREG + +# GCN-LABEL: bb.2: +# GCN: S_SETREG +# GCN-NEXT: S_SETREG + +name: s_setreg + +body: | + bb.0: + successors: %bb.1 + S_SETREG_B32 %sgpr0, 1 + S_SETREG_B32 %sgpr1, 1 + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + S_SETREG_B32 %sgpr0, 64 + S_SETREG_B32 %sgpr1, 128 + S_BRANCH %bb.2 + + bb.2: + S_SETREG_B32 %sgpr0, 1 + S_SETREG_B32 %sgpr1, 0 + S_ENDPGM +... + +... +--- +# GCN-LABEL: name: vmem_gt_8dw_store + +# GCN-LABEL: bb.0: +# GCN: BUFFER_STORE_DWORD_OFFSET +# GCN-NEXT: V_MOV_B32 +# GCN: BUFFER_STORE_DWORDX3_OFFSET +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: BUFFER_STORE_DWORDX4_OFFSET +# GCN-NEXT: V_MOV_B32 +# GCN: BUFFER_STORE_DWORDX4_OFFSET +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: BUFFER_STORE_FORMAT_XYZ_OFFSET +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: BUFFER_STORE_FORMAT_XYZW_OFFSET +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 + +# GCN-LABEL: bb.1: +# GCN: FLAT_STORE_DWORDX2 +# GCN-NEXT: V_MOV_B32 +# GCN: FLAT_STORE_DWORDX3 +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: FLAT_STORE_DWORDX4 +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: FLAT_ATOMIC_CMPSWAP_X2 +# CIVI: S_NOP +# GCN-NEXT: V_MOV_B32 +# GCN: FLAT_ATOMIC_FCMPSWAP_X2 +# CIVI: S_NOP +# GCN: V_MOV_B32 + +name: vmem_gt_8dw_store + +body: | + bb.0: + successors: %bb.1 + BUFFER_STORE_DWORD_OFFSET %vgpr3, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_STORE_DWORDX3_OFFSET %vgpr2_vgpr3_vgpr4, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_STORE_DWORDX4_OFFSET %vgpr2_vgpr3_vgpr4_vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, %sgpr4, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_STORE_DWORDX4_OFFSET %vgpr2_vgpr3_vgpr4_vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_STORE_FORMAT_XYZ_OFFSET %vgpr2_vgpr3_vgpr4, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_STORE_FORMAT_XYZW_OFFSET %vgpr2_vgpr3_vgpr4_vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + BUFFER_ATOMIC_CMPSWAP_X2_OFFSET %vgpr2_vgpr3_vgpr4_vgpr5, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, implicit %exec + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + S_BRANCH %bb.1 + + bb.1: + FLAT_STORE_DWORDX2 %vgpr0_vgpr1, %vgpr2_vgpr3, 0, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + FLAT_STORE_DWORDX3 %vgpr0_vgpr1, %vgpr2_vgpr3_vgpr4, 0, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + FLAT_STORE_DWORDX4 %vgpr0_vgpr1, %vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + FLAT_ATOMIC_CMPSWAP_X2 %vgpr0_vgpr1, %vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + FLAT_ATOMIC_FCMPSWAP_X2 %vgpr0_vgpr1, %vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3 = V_MOV_B32_e32 0, implicit %exec + S_ENDPGM + +... + +... +--- + +# GCN-LABEL: name: readwrite_lane + +# GCN-LABEL: bb.0: +# GCN: V_ADD_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_READLANE_B32 + +# GCN-LABEL: bb.1: +# GCN: V_ADD_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_WRITELANE_B32 + +# GCN-LABEL: bb.2: +# GCN: V_ADD_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_READLANE_B32 + +# GCN-LABEL: bb.3: +# GCN: V_ADD_I32 +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: S_NOP +# GCN: V_WRITELANE_B32 + +name: readwrite_lane + +body: | + bb.0: + successors: %bb.1 + %vgpr0,%sgpr0_sgpr1 = V_ADD_I32_e64 %vgpr1, %vgpr2, implicit %vcc, implicit %exec + %sgpr4 = V_READLANE_B32 %vgpr4, %sgpr0 + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + %vgpr0,%sgpr0_sgpr1 = V_ADD_I32_e64 %vgpr1, %vgpr2, implicit %vcc, implicit %exec + %vgpr4 = V_WRITELANE_B32 %sgpr0, %sgpr0 + S_BRANCH %bb.2 + + bb.2: + successors: %bb.3 + %vgpr0,implicit %vcc = V_ADD_I32_e32 %vgpr1, %vgpr2, implicit %vcc, implicit %exec + %sgpr4 = V_READLANE_B32 %vgpr4, %vcc_lo + S_BRANCH %bb.3 + + bb.3: + %vgpr0,implicit %vcc = V_ADD_I32_e32 %vgpr1, %vgpr2, implicit %vcc, implicit %exec + %vgpr4 = V_WRITELANE_B32 %sgpr4, %vcc_lo + S_ENDPGM + +... + +... +--- + +# GCN-LABEL: name: rfe + +# GCN-LABEL: bb.0: +# GCN: S_SETREG +# VI: S_NOP +# GCN-NEXT: S_RFE_B64 + +# GCN-LABEL: bb.1: +# GCN: S_SETREG +# GCN-NEXT: S_RFE_B64 + +name: rfe + +body: | + bb.0: + successors: %bb.1 + S_SETREG_B32 %sgpr0, 3 + S_RFE_B64 %sgpr2_sgpr3 + S_BRANCH %bb.1 + + bb.1: + S_SETREG_B32 %sgpr0, 0 + S_RFE_B64 %sgpr2_sgpr3 + S_ENDPGM + +... diff --git a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata1.ll b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata1.ll index 4e17a921d91b..49c314fbc5d0 100644 --- a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata1.ll +++ b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata1.ll @@ -1,8 +1,6 @@ -; RUN: llc -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-runtime-metadata | FileCheck %s ; check llc does not crash for invalid opencl version metadata -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .short 256 +; CHECK: { amd.MDVersion: [ 2, 0 ] } !opencl.ocl.version = !{} diff --git a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata2.ll b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata2.ll index 35b7d70596c1..1f5e8be531dc 100644 --- a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata2.ll +++ b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata2.ll @@ -1,9 +1,7 @@ -; RUN: llc -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-runtime-metadata | FileCheck %s ; check llc does not crash for invalid opencl version metadata -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .short 256 +; CHECK: { amd.MDVersion: [ 2, 0 ] } !opencl.ocl.version = !{!0} !0 = !{} diff --git a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata3.ll b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata3.ll index e1693551b621..b77551e268a0 100644 --- a/test/CodeGen/AMDGPU/invalid-opencl-version-metadata3.ll +++ b/test/CodeGen/AMDGPU/invalid-opencl-version-metadata3.ll @@ -1,9 +1,7 @@ -; RUN: llc -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-runtime-metadata | FileCheck %s ; check llc does not crash for invalid opencl version metadata -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .short 256 +; CHECK: { amd.MDVersion: [ 2, 0 ] } !opencl.ocl.version = !{!0} !0 = !{i32 1} diff --git a/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll b/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll index 347170f79e32..45a061067cfc 100644 --- a/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll +++ b/test/CodeGen/AMDGPU/invariant-load-no-alias-store.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -amdgpu-load-store-vectorizer=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; GatherAllAliases gives up on trying to analyze cases where the ; pointer may have been loaded from an aliased store, so make sure diff --git a/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir b/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir new file mode 100644 index 000000000000..66182d092895 --- /dev/null +++ b/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir @@ -0,0 +1,89 @@ +# RUN: llc -run-pass block-placement -march=amdgcn -verify-machineinstrs -o - %s | FileCheck %s +--- | + + define void @invert_br_undef_vcc(float %cond, i32 addrspace(1)* %out) #0 { + entry: + br i1 undef, label %if, label %else, !structurizecfg.uniform !0, !amdgpu.uniform !0 + + else: ; preds = %entry + store volatile i32 100, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + if: ; preds = %entry + store volatile i32 9, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + done: ; preds = %if, %else + %value = phi i32 [ 0, %if ], [ 1, %else ] + store i32 %value, i32 addrspace(1)* %out + ret void + } + + attributes #0 = { nounwind } + + !0 = !{} + +... +--- +# CHECK-LABEL: name: invert_br_undef_vcc +# CHECK: S_CBRANCH_VCCZ %bb.1.else, implicit undef %vcc + +name: invert_br_undef_vcc +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%sgpr0_sgpr1' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.entry: + successors: %bb.2.if, %bb.1.else + liveins: %sgpr0_sgpr1 + + %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`) + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + S_CBRANCH_VCCNZ %bb.2.if, implicit undef %vcc + + bb.1.else: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 100, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 1, implicit %exec + S_BRANCH %bb.3.done + + bb.2.if: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 9, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 0, implicit %exec + + bb.3.done: + liveins: %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into %ir.out) + S_ENDPGM + +... diff --git a/test/CodeGen/AMDGPU/kernel-args.ll b/test/CodeGen/AMDGPU/kernel-args.ll index 7567b38e0cea..b1f20fd995fd 100644 --- a/test/CodeGen/AMDGPU/kernel-args.ll +++ b/test/CodeGen/AMDGPU/kernel-args.ll @@ -1,13 +1,21 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=GCN --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=VI --check-prefix=GCN --check-prefix=FUNC +; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefixes=SI,GCN,MESA-GCN,FUNC +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefixes=VI,GCN,MESA-VI,MESA-GCN,FUNC +; RUN: llc < %s -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefixes=VI,GCN,HSA-VI,FUNC ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC ; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=EG --check-prefix=FUNC ; FUNC-LABEL: {{^}}i8_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: AND_INT {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb -; VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; GCN: s_and_b32 s{{[0-9]+}}, [[VAL]], 0xff +; MESA-VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c +; MESA-GCN: s_and_b32 s{{[0-9]+}}, [[VAL]], 0xff +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_ubyte v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i8_arg(i32 addrspace(1)* nocapture %out, i8 %in) nounwind { entry: @@ -17,9 +25,16 @@ entry: } ; FUNC-LABEL: {{^}}i8_zext_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: MOV {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_ubyte v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i8_zext_arg(i32 addrspace(1)* nocapture %out, i8 zeroext %in) nounwind { entry: @@ -29,9 +44,16 @@ entry: } ; FUNC-LABEL: {{^}}i8_sext_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: MOV {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_sbyte v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i8_sext_arg(i32 addrspace(1)* nocapture %out, i8 signext %in) nounwind { entry: @@ -41,10 +63,17 @@ entry: } ; FUNC-LABEL: {{^}}i16_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: AND_INT {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb -; VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c -; GCN: s_and_b32 s{{[0-9]+}}, [[VAL]], 0xff +; MESA-VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x2c +; MESA-GCN: s_and_b32 s{{[0-9]+}}, [[VAL]], 0xff +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_ushort v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i16_arg(i32 addrspace(1)* nocapture %out, i16 %in) nounwind { entry: @@ -54,9 +83,16 @@ entry: } ; FUNC-LABEL: {{^}}i16_zext_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: MOV {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_ushort v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i16_zext_arg(i32 addrspace(1)* nocapture %out, i16 zeroext %in) nounwind { entry: @@ -66,9 +102,16 @@ entry: } ; FUNC-LABEL: {{^}}i16_sext_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: MOV {{[ *]*}}T{{[0-9]+\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_add_u32 [[SPTR_LO:s[0-9]+]], s4, 8 +; HSA-VI: s_addc_u32 [[SPTR_HI:s[0-9]+]], s5, 0 +; HSA-VI: v_mov_b32_e32 v[[VPTR_LO:[0-9]+]], [[SPTR_LO]] +; HSA-VI: v_mov_b32_e32 v[[VPTR_HI:[0-9]+]], [[SPTR_HI]] +; FIXME: Should be using s_load_dword +; HSA-VI: flat_load_sshort v{{[0-9]+}}, v{{\[}}[[VPTR_LO]]:[[VPTR_HI]]] define void @i16_sext_arg(i32 addrspace(1)* nocapture %out, i16 signext %in) nounwind { entry: @@ -78,9 +121,11 @@ entry: } ; FUNC-LABEL: {{^}}i32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: T{{[0-9]\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_load_dword s{{[0-9]}}, s[4:5], 0x8 define void @i32_arg(i32 addrspace(1)* nocapture %out, i32 %in) nounwind { entry: store i32 %in, i32 addrspace(1)* %out, align 4 @@ -88,9 +133,11 @@ entry: } ; FUNC-LABEL: {{^}}f32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: T{{[0-9]\.[XYZW]}}, KC0[2].Z ; SI: s_load_dword s{{[0-9]}}, s[0:1], 0xb -; VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; MESA-VI: s_load_dword s{{[0-9]}}, s[0:1], 0x2c +; HSA-VI: s_load_dword s{{[0-9]+}}, s[4:5], 0x8 define void @f32_arg(float addrspace(1)* nocapture %out, float %in) nounwind { entry: store float %in, float addrspace(1)* %out, align 4 @@ -98,10 +145,13 @@ entry: } ; FUNC-LABEL: {{^}}v2i8_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_8 ; EG: VTX_READ_8 -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte define void @v2i8_arg(<2 x i8> addrspace(1)* %out, <2 x i8> %in) { entry: store <2 x i8> %in, <2 x i8> addrspace(1)* %out @@ -109,10 +159,13 @@ entry: } ; FUNC-LABEL: {{^}}v2i16_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_16 ; EG: VTX_READ_16 -; GCN-DAG: buffer_load_ushort -; GCN-DAG: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort define void @v2i16_arg(<2 x i16> addrspace(1)* %out, <2 x i16> %in) { entry: store <2 x i16> %in, <2 x i16> addrspace(1)* %out @@ -120,10 +173,12 @@ entry: } ; FUNC-LABEL: {{^}}v2i32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].X ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[2].W ; SI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0xb -; VI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x2c +; MESA-VI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x2c +; HSA-VI: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x8 define void @v2i32_arg(<2 x i32> addrspace(1)* nocapture %out, <2 x i32> %in) nounwind { entry: store <2 x i32> %in, <2 x i32> addrspace(1)* %out, align 4 @@ -131,10 +186,12 @@ entry: } ; FUNC-LABEL: {{^}}v2f32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].X ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[2].W ; SI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0xb -; VI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x2c +; MESA-VI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x2c +; HSA-VI: s_load_dwordx2 s{{\[[0-9]:[0-9]\]}}, s[4:5], 0x8 define void @v2f32_arg(<2 x float> addrspace(1)* nocapture %out, <2 x float> %in) nounwind { entry: store <2 x float> %in, <2 x float> addrspace(1)* %out, align 4 @@ -142,9 +199,16 @@ entry: } ; FUNC-LABEL: {{^}}v3i8_arg: -; VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 40 -; VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 41 -; VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 42 +; HSA-VI: kernarg_segment_alignment = 4 +; EG-DAG: VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 40 +; EG-DAG: VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 41 +; EG-DAG: VTX_READ_8 T{{[0-9]}}.X, T{{[0-9]}}.X, 42 +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte define void @v3i8_arg(<3 x i8> addrspace(1)* nocapture %out, <3 x i8> %in) nounwind { entry: store <3 x i8> %in, <3 x i8> addrspace(1)* %out, align 4 @@ -152,20 +216,29 @@ entry: } ; FUNC-LABEL: {{^}}v3i16_arg: -; VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 44 -; VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 46 -; VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 48 +; HSA-VI: kernarg_segment_alignment = 4 +; EG-DAG: VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 44 +; EG-DAG: VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 46 +; EG-DAG: VTX_READ_16 T{{[0-9]}}.X, T{{[0-9]}}.X, 48 +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort define void @v3i16_arg(<3 x i16> addrspace(1)* nocapture %out, <3 x i16> %in) nounwind { entry: store <3 x i16> %in, <3 x i16> addrspace(1)* %out, align 4 ret void } ; FUNC-LABEL: {{^}}v3i32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].W ; SI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0xd -; VI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0x34 +; MESA-VI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0x34 +; HSA-VI: s_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x10 define void @v3i32_arg(<3 x i32> addrspace(1)* nocapture %out, <3 x i32> %in) nounwind { entry: store <3 x i32> %in, <3 x i32> addrspace(1)* %out, align 4 @@ -173,11 +246,13 @@ entry: } ; FUNC-LABEL: {{^}}v3f32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].W ; SI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0xd -; VI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0x34 +; MESA-VI: s_load_dwordx4 s{{\[[0-9]:[0-9]+\]}}, s[0:1], 0x34 +; HSA-VI: s_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x10 define void @v3f32_arg(<3 x float> addrspace(1)* nocapture %out, <3 x float> %in) nounwind { entry: store <3 x float> %in, <3 x float> addrspace(1)* %out, align 4 @@ -185,14 +260,19 @@ entry: } ; FUNC-LABEL: {{^}}v4i8_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte define void @v4i8_arg(<4 x i8> addrspace(1)* %out, <4 x i8> %in) { entry: store <4 x i8> %in, <4 x i8> addrspace(1)* %out @@ -200,14 +280,19 @@ entry: } ; FUNC-LABEL: {{^}}v4i16_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; HSA-GCN: flat_load_ushort +; HSA-GCN: flat_load_ushort +; HSA-GCN: flat_load_ushort +; HSA-GCN: flat_load_ushort define void @v4i16_arg(<4 x i16> addrspace(1)* %out, <4 x i16> %in) { entry: store <4 x i16> %in, <4 x i16> addrspace(1)* %out @@ -215,12 +300,14 @@ entry: } ; FUNC-LABEL: {{^}}v4i32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].W ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].X ; SI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0xd -; VI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x34 +; MESA-VI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x34 +; HSA-VI: s_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x10 define void @v4i32_arg(<4 x i32> addrspace(1)* nocapture %out, <4 x i32> %in) nounwind { entry: store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4 @@ -228,12 +315,14 @@ entry: } ; FUNC-LABEL: {{^}}v4f32_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[3].W ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].X ; SI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0xd -; VI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x34 +; MESA-VI: s_load_dwordx4 s{{\[[0-9]:[0-9]\]}}, s[0:1], 0x34 +; HSA-VI: s_load_dwordx4 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x10 define void @v4f32_arg(<4 x float> addrspace(1)* nocapture %out, <4 x float> %in) nounwind { entry: store <4 x float> %in, <4 x float> addrspace(1)* %out, align 4 @@ -241,6 +330,7 @@ entry: } ; FUNC-LABEL: {{^}}v8i8_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 @@ -249,13 +339,21 @@ entry: ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte +; HSA-GCN: float_load_ubyte define void @v8i8_arg(<8 x i8> addrspace(1)* %out, <8 x i8> %in) { entry: store <8 x i8> %in, <8 x i8> addrspace(1)* %out @@ -263,6 +361,7 @@ entry: } ; FUNC-LABEL: {{^}}v8i16_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 @@ -271,14 +370,22 @@ entry: ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort define void @v8i16_arg(<8 x i16> addrspace(1)* %out, <8 x i16> %in) { entry: store <8 x i16> %in, <8 x i16> addrspace(1)* %out @@ -286,6 +393,7 @@ entry: } ; FUNC-LABEL: {{^}}v8i32_arg: +; HSA-VI: kernarg_segment_alignment = 5 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].W @@ -295,7 +403,8 @@ entry: ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[5].W ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].X ; SI: s_load_dwordx8 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x11 -; VI: s_load_dwordx8 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x44 +; MESA-VI: s_load_dwordx8 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x44 +; HSA-VI: s_load_dwordx8 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x20 define void @v8i32_arg(<8 x i32> addrspace(1)* nocapture %out, <8 x i32> %in) nounwind { entry: store <8 x i32> %in, <8 x i32> addrspace(1)* %out, align 4 @@ -303,6 +412,7 @@ entry: } ; FUNC-LABEL: {{^}}v8f32_arg: +; HSA-VI: kernarg_segment_alignment = 5 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[4].W @@ -319,6 +429,7 @@ entry: } ; FUNC-LABEL: {{^}}v16i8_arg: +; HSA-VI: kernarg_segment_alignment = 4 ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 @@ -335,22 +446,38 @@ entry: ; EG: VTX_READ_8 ; EG: VTX_READ_8 ; EG: VTX_READ_8 -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte -; GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; MESA-GCN: buffer_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte +; HSA-VI: flat_load_ubyte define void @v16i8_arg(<16 x i8> addrspace(1)* %out, <16 x i8> %in) { entry: store <16 x i8> %in, <16 x i8> addrspace(1)* %out @@ -358,6 +485,7 @@ entry: } ; FUNC-LABEL: {{^}}v16i16_arg: +; HSA-VI: kernarg_segment_alignment = 5 ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 @@ -374,22 +502,38 @@ entry: ; EG: VTX_READ_16 ; EG: VTX_READ_16 ; EG: VTX_READ_16 -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort -; GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; MESA-GCN: buffer_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort +; HSA-VI: flat_load_ushort define void @v16i16_arg(<16 x i16> addrspace(1)* %out, <16 x i16> %in) { entry: store <16 x i16> %in, <16 x i16> addrspace(1)* %out @@ -397,6 +541,7 @@ entry: } ; FUNC-LABEL: {{^}}v16i32_arg: +; HSA-VI: kernarg_segment_alignment = 6 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].W @@ -414,7 +559,8 @@ entry: ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[9].W ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[10].X ; SI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x19 -; VI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x64 +; MESA-VI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x64 +; HSA-VI: s_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x40 define void @v16i32_arg(<16 x i32> addrspace(1)* nocapture %out, <16 x i32> %in) nounwind { entry: store <16 x i32> %in, <16 x i32> addrspace(1)* %out, align 4 @@ -422,6 +568,7 @@ entry: } ; FUNC-LABEL: {{^}}v16f32_arg: +; HSA-VI: kernarg_segment_alignment = 6 ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].Y ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].Z ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[6].W @@ -439,7 +586,8 @@ entry: ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[9].W ; EG-DAG: T{{[0-9]\.[XYZW]}}, KC0[10].X ; SI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x19 -; VI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x64 +; MESA-VI: s_load_dwordx16 s{{\[[0-9]+:[0-9]+\]}}, s[0:1], 0x64 +; HSA-VI: s_load_dwordx16 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x40 define void @v16f32_arg(<16 x float> addrspace(1)* nocapture %out, <16 x float> %in) nounwind { entry: store <16 x float> %in, <16 x float> addrspace(1)* %out, align 4 @@ -447,9 +595,10 @@ entry: } ; FUNC-LABEL: {{^}}kernel_arg_i64: -; GCN: s_load_dwordx2 -; GCN: s_load_dwordx2 -; GCN: buffer_store_dwordx2 +; MESA-GCN: s_load_dwordx2 +; MESA-GCN: s_load_dwordx2 +; MESA-GCN: buffer_store_dwordx2 +; HSA-VI: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x8 define void @kernel_arg_i64(i64 addrspace(1)* %out, i64 %a) nounwind { store i64 %a, i64 addrspace(1)* %out, align 8 ret void @@ -458,9 +607,10 @@ define void @kernel_arg_i64(i64 addrspace(1)* %out, i64 %a) nounwind { ; FUNC-LABEL: {{^}}f64_kernel_arg: ; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x9 ; SI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0xb -; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x24 -; VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x2c -; GCN: buffer_store_dwordx2 +; MESA-VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x24 +; MESA-VI-DAG: s_load_dwordx2 s[{{[0-9]:[0-9]}}], s[0:1], 0x2c +; MESA-GCN: buffer_store_dwordx2 +; HSA-VI: s_load_dwordx2 s[{{[0-9]+:[0-9]+}}], s[4:5], 0x8 define void @f64_kernel_arg(double addrspace(1)* %out, double %in) { entry: store double %in, double addrspace(1)* %out diff --git a/test/CodeGen/AMDGPU/large-alloca-compute.ll b/test/CodeGen/AMDGPU/large-alloca-compute.ll index 099f0639b34c..4f6dbf9dc2bf 100644 --- a/test/CodeGen/AMDGPU/large-alloca-compute.ll +++ b/test/CodeGen/AMDGPU/large-alloca-compute.ll @@ -18,13 +18,13 @@ ; GCNHSA: .amd_kernel_code_t -; GCNHSA: compute_pgm_rsrc2_scratch_en = 1 -; GCNHSA: compute_pgm_rsrc2_user_sgpr = 8 -; GCNHSA: compute_pgm_rsrc2_tgid_x_en = 1 -; GCNHSA: compute_pgm_rsrc2_tgid_y_en = 0 -; GCNHSA: compute_pgm_rsrc2_tgid_z_en = 0 -; GCNHSA: compute_pgm_rsrc2_tg_size_en = 0 -; GCNHSA: compute_pgm_rsrc2_tidig_comp_cnt = 0 +; GCNHSA: enable_sgpr_private_segment_wave_byte_offset = 1 +; GCNHSA: user_sgpr_count = 8 +; GCNHSA: enable_sgpr_workgroup_id_x = 1 +; GCNHSA: enable_sgpr_workgroup_id_y = 0 +; GCNHSA: enable_sgpr_workgroup_id_z = 0 +; GCNHSA: enable_sgpr_workgroup_info = 0 +; GCNHSA: enable_vgpr_workitem_id = 0 ; GCNHSA: enable_sgpr_private_segment_buffer = 1 ; GCNHSA: enable_sgpr_dispatch_ptr = 0 diff --git a/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll b/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll index f661939214c0..906a688febd2 100644 --- a/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll +++ b/test/CodeGen/AMDGPU/large-work-group-promote-alloca.ll @@ -66,7 +66,7 @@ entry: ret void } -; CHECK: @occupancy_0( +; CHECK-LABEL: @occupancy_0( ; CHECK: alloca [5 x i32] define void @occupancy_0(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #3 { entry: @@ -88,7 +88,7 @@ entry: ret void } -; CHECK: @occupancy_max( +; CHECK-LABEL: @occupancy_max( ; CHECK: alloca [5 x i32] define void @occupancy_max(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* nocapture %in) #4 { entry: @@ -110,8 +110,155 @@ entry: ret void } +; CHECK-LABEL: @occupancy_6( +; CHECK-NOT: alloca +define void @occupancy_6(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #5 { +entry: + %stack = alloca [42 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [42 x i8], [42 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [42 x i8], [42 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [42 x i8], [42 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [42 x i8], [42 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + +; CHECK-LABEL: @occupancy_6_over( +; CHECK: alloca [43 x i8] +define void @occupancy_6_over(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #5 { +entry: + %stack = alloca [43 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [43 x i8], [43 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [43 x i8], [43 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [43 x i8], [43 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [43 x i8], [43 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + +; CHECK-LABEL: @occupancy_8( +; CHECK-NOT: alloca +define void @occupancy_8(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #6 { +entry: + %stack = alloca [32 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [32 x i8], [32 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [32 x i8], [32 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [32 x i8], [32 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [32 x i8], [32 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + +; CHECK-LABEL: @occupancy_8_over( +; CHECK: alloca [33 x i8] +define void @occupancy_8_over(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #6 { +entry: + %stack = alloca [33 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [33 x i8], [33 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [33 x i8], [33 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [33 x i8], [33 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [33 x i8], [33 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + +; CHECK-LABEL: @occupancy_9( +; CHECK-NOT: alloca +define void @occupancy_9(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #7 { +entry: + %stack = alloca [28 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [28 x i8], [28 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [28 x i8], [28 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [28 x i8], [28 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [28 x i8], [28 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + +; CHECK-LABEL: @occupancy_9_over( +; CHECK: alloca [29 x i8] +define void @occupancy_9_over(i8 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %in) #7 { +entry: + %stack = alloca [29 x i8], align 4 + %tmp = load i8, i8 addrspace(1)* %in, align 1 + %tmp4 = sext i8 %tmp to i64 + %arrayidx1 = getelementptr inbounds [29 x i8], [29 x i8]* %stack, i64 0, i64 %tmp4 + store i8 4, i8* %arrayidx1, align 1 + %arrayidx2 = getelementptr inbounds i8, i8 addrspace(1)* %in, i64 1 + %tmp1 = load i8, i8 addrspace(1)* %arrayidx2, align 1 + %tmp5 = sext i8 %tmp1 to i64 + %arrayidx3 = getelementptr inbounds [29 x i8], [29 x i8]* %stack, i64 0, i64 %tmp5 + store i8 5, i8* %arrayidx3, align 1 + %arrayidx10 = getelementptr inbounds [29 x i8], [29 x i8]* %stack, i64 0, i64 0 + %tmp2 = load i8, i8* %arrayidx10, align 1 + store i8 %tmp2, i8 addrspace(1)* %out, align 1 + %arrayidx12 = getelementptr inbounds [29 x i8], [29 x i8]* %stack, i64 0, i64 1 + %tmp3 = load i8, i8* %arrayidx12, align 1 + %arrayidx13 = getelementptr inbounds i8, i8 addrspace(1)* %out, i64 1 + store i8 %tmp3, i8 addrspace(1)* %arrayidx13, align 1 + ret void +} + attributes #0 = { nounwind "amdgpu-max-work-group-size"="63" } -attributes #1 = { nounwind "amdgpu-max-waves-per-eu"="3" "amdgpu-max-work-group-size"="256" } -attributes #2 = { nounwind "amdgpu-max-waves-per-eu"="1" "amdgpu-max-work-group-size"="1600" } -attributes #3 = { nounwind "amdgpu-max-waves-per-eu"="0" } -attributes #4 = { nounwind "amdgpu-max-waves-per-eu"="-1" } +attributes #1 = { nounwind "amdgpu-waves-per-eu"="1,3" "amdgpu-flat-work-group-size"="256,256" } +attributes #2 = { nounwind "amdgpu-waves-per-eu"="1,1" "amdgpu-flat-work-group-size"="1600,1600" } +attributes #3 = { nounwind "amdgpu-waves-per-eu"="1,10" } +attributes #4 = { nounwind "amdgpu-waves-per-eu"="1,10" } +attributes #5 = { nounwind "amdgpu-waves-per-eu"="1,6" "amdgpu-flat-work-group-size"="64,64" } +attributes #6 = { nounwind "amdgpu-waves-per-eu"="1,8" "amdgpu-flat-work-group-size"="64,64" } +attributes #7 = { nounwind "amdgpu-waves-per-eu"="1,9" "amdgpu-flat-work-group-size"="64,64" } diff --git a/test/CodeGen/AMDGPU/large-work-group-registers.ll b/test/CodeGen/AMDGPU/large-work-group-registers.ll deleted file mode 100644 index 468633da56d8..000000000000 --- a/test/CodeGen/AMDGPU/large-work-group-registers.ll +++ /dev/null @@ -1,41 +0,0 @@ -; RUN: llc -march=amdgcn -mcpu=tonga -post-RA-scheduler=0 < %s | FileCheck %s - -; CHECK: NumVgprs: 64 -define void @main([9 x <16 x i8>] addrspace(2)* byval, [17 x <16 x i8>] addrspace(2)* byval, [17 x <8 x i32>] addrspace(2)* byval, [16 x <8 x i32>] addrspace(2)* byval, [16 x <4 x i32>] addrspace(2)* byval, <3 x i32> inreg, <3 x i32> inreg, <3 x i32>) #0 { -main_body: - %8 = getelementptr [16 x <4 x i32>], [16 x <4 x i32>] addrspace(2)* %4, i64 0, i64 8 - %9 = load <4 x i32>, <4 x i32> addrspace(2)* %8, align 16, !tbaa !0 - %10 = extractelement <3 x i32> %7, i32 0 - %11 = extractelement <3 x i32> %7, i32 1 - %12 = mul i32 %10, %11 - %bc = bitcast <3 x i32> %7 to <3 x float> - %13 = extractelement <3 x float> %bc, i32 1 - %14 = insertelement <512 x float> undef, float %13, i32 %12 - call void @llvm.amdgcn.s.barrier() - %15 = extractelement <3 x i32> %6, i32 0 - %16 = extractelement <3 x i32> %7, i32 0 - %17 = shl i32 %15, 5 - %18 = add i32 %17, %16 - %19 = shl i32 %18, 4 - %20 = extractelement <3 x i32> %7, i32 1 - %21 = shl i32 %20, 2 - %22 = sext i32 %21 to i64 - %23 = getelementptr i8, i8 addrspace(3)* null, i64 %22 - %24 = bitcast i8 addrspace(3)* %23 to i32 addrspace(3)* - %25 = load i32, i32 addrspace(3)* %24, align 4 - %26 = extractelement <512 x float> %14, i32 %25 - %27 = insertelement <4 x float> undef, float %26, i32 0 - call void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float> %27, <4 x i32> %9, i32 0, i32 %19, i1 false, i1 false) - ret void -} - -declare void @llvm.amdgcn.s.barrier() #1 - -declare void @llvm.amdgcn.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #2 - -attributes #0 = { "amdgpu-max-work-group-size"="1024" } -attributes #1 = { convergent nounwind } -attributes #2 = { nounwind } - -!0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} diff --git a/test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll b/test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll index 0c734c6d99dd..078d6330ce04 100644 --- a/test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll +++ b/test/CodeGen/AMDGPU/lds-m0-init-in-loop.ll @@ -9,10 +9,12 @@ ; GCN: s_mov_b32 m0, -1 ; GCN: BB0_2: +; GCN-NOT: m0 ; GCN: ds_read_b32 +; GCN-NOT: m0 ; GCN: buffer_store_dword -; GCN: s_cbranch_vccz BB0_2 +; GCN: s_cbranch_scc0 BB0_2 ; GCN: BB0_3: ; GCN-NEXT: s_endpgm diff --git a/test/CodeGen/AMDGPU/liveness.mir b/test/CodeGen/AMDGPU/liveness.mir index ce49294d5b36..112c3f8e69a6 100644 --- a/test/CodeGen/AMDGPU/liveness.mir +++ b/test/CodeGen/AMDGPU/liveness.mir @@ -17,16 +17,16 @@ registers: body: | bb.0: successors: %bb.1, %bb.2 - S_NOP 0, implicit-def undef %0:sub0 + S_NOP 0, implicit-def undef %0.sub0 S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc S_BRANCH %bb.2 bb.1: successors: %bb.2 - S_NOP 0, implicit-def %0:sub1 - S_NOP 0, implicit %0:sub1 + S_NOP 0, implicit-def %0.sub1 + S_NOP 0, implicit %0.sub1 S_BRANCH %bb.2 bb.2: - S_NOP 0, implicit %0:sub0 + S_NOP 0, implicit %0.sub0 ... diff --git a/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll index 541119242a94..bf5d492dca41 100644 --- a/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll +++ b/test/CodeGen/AMDGPU/llvm.AMDGPU.bfe.u32.ll @@ -1,5 +1,6 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=FUNC -check-prefix=GCN %s ; RUN: llc -march=r600 -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s declare i32 @llvm.AMDGPU.bfe.u32(i32, i32, i32) nounwind readnone @@ -73,11 +74,14 @@ define void @bfe_u32_zextload_i8(i32 addrspace(1)* %out, i8 addrspace(1)* %in) n } ; FUNC-LABEL: {{^}}bfe_u32_zext_in_reg_i8: -; SI: buffer_load_dword +; GCN: buffer_load_dword ; SI: v_add_i32 ; SI-NEXT: v_and_b32_e32 +; FIXME: Should be using s_add_i32 +; VI: v_add_i32 +; VI-NEXT: v_and_b32_e32 ; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm +; GCN: s_endpgm define void @bfe_u32_zext_in_reg_i8(i32 addrspace(1)* %out, i32 addrspace(1)* %in) nounwind { %load = load i32, i32 addrspace(1)* %in, align 4 %add = add i32 %load, 1 diff --git a/test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll index cfe4cc00ee81..88e9c6529778 100644 --- a/test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll +++ b/test/CodeGen/AMDGPU/llvm.AMDGPU.clamp.ll @@ -7,7 +7,7 @@ declare float @llvm.AMDGPU.clamp.f32(float, float, float) nounwind readnone ; FUNC-LABEL: {{^}}clamp_0_1_f32: ; SI: s_load_dword [[ARG:s[0-9]+]], -; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], 0, [[ARG]] clamp{{$}} +; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], [[ARG]], 0 clamp{{$}} ; SI: buffer_store_dword [[RESULT]] ; SI: s_endpgm @@ -20,7 +20,7 @@ define void @clamp_0_1_f32(float addrspace(1)* %out, float %src) nounwind { ; FUNC-LABEL: {{^}}clamp_fabs_0_1_f32: ; SI: s_load_dword [[ARG:s[0-9]+]], -; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], 0, |[[ARG]]| clamp{{$}} +; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], |[[ARG]]|, 0 clamp{{$}} ; SI: buffer_store_dword [[RESULT]] ; SI: s_endpgm define void @clamp_fabs_0_1_f32(float addrspace(1)* %out, float %src) nounwind { @@ -32,7 +32,7 @@ define void @clamp_fabs_0_1_f32(float addrspace(1)* %out, float %src) nounwind { ; FUNC-LABEL: {{^}}clamp_fneg_0_1_f32: ; SI: s_load_dword [[ARG:s[0-9]+]], -; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], 0, -[[ARG]] clamp{{$}} +; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], -[[ARG]], 0 clamp{{$}} ; SI: buffer_store_dword [[RESULT]] ; SI: s_endpgm define void @clamp_fneg_0_1_f32(float addrspace(1)* %out, float %src) nounwind { @@ -44,7 +44,7 @@ define void @clamp_fneg_0_1_f32(float addrspace(1)* %out, float %src) nounwind { ; FUNC-LABEL: {{^}}clamp_fneg_fabs_0_1_f32: ; SI: s_load_dword [[ARG:s[0-9]+]], -; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], 0, -|[[ARG]]| clamp{{$}} +; SI: v_add_f32_e64 [[RESULT:v[0-9]+]], -|[[ARG]]|, 0 clamp{{$}} ; SI: buffer_store_dword [[RESULT]] ; SI: s_endpgm define void @clamp_fneg_fabs_0_1_f32(float addrspace(1)* %out, float %src) nounwind { diff --git a/test/CodeGen/AMDGPU/llvm.AMDGPU.flbit.i32.ll b/test/CodeGen/AMDGPU/llvm.AMDGPU.flbit.i32.ll deleted file mode 100644 index 20c7af8ade5e..000000000000 --- a/test/CodeGen/AMDGPU/llvm.AMDGPU.flbit.i32.ll +++ /dev/null @@ -1,28 +0,0 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s - -declare i32 @llvm.AMDGPU.flbit.i32(i32) nounwind readnone - -; FUNC-LABEL: {{^}}s_flbit: -; SI: s_load_dword [[VAL:s[0-9]+]], -; SI: s_flbit_i32 [[SRESULT:s[0-9]+]], [[VAL]] -; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] -; SI: buffer_store_dword [[VRESULT]], -; SI: s_endpgm -define void @s_flbit(i32 addrspace(1)* noalias %out, i32 %val) nounwind { - %r = call i32 @llvm.AMDGPU.flbit.i32(i32 %val) nounwind readnone - store i32 %r, i32 addrspace(1)* %out, align 4 - ret void -} - -; FUNC-LABEL: {{^}}v_flbit: -; SI: buffer_load_dword [[VAL:v[0-9]+]], -; SI: v_ffbh_i32_e32 [[RESULT:v[0-9]+]], [[VAL]] -; SI: buffer_store_dword [[RESULT]], -; SI: s_endpgm -define void @v_flbit(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { - %val = load i32, i32 addrspace(1)* %valptr, align 4 - %r = call i32 @llvm.AMDGPU.flbit.i32(i32 %val) nounwind readnone - store i32 %r, i32 addrspace(1)* %out, align 4 - ret void -} diff --git a/test/CodeGen/AMDGPU/llvm.SI.export.ll b/test/CodeGen/AMDGPU/llvm.SI.export.ll new file mode 100644 index 000000000000..23a32dcfd943 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.SI.export.ll @@ -0,0 +1,237 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -strict-whitespace -check-prefix=GCN %s + +declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) #0 + +; GCN-LABEL: {{^}}test_export_zeroes: +; GCN: exp mrt0 off, off, off, off{{$}} +; GCN: exp mrt0 off, off, off, off done{{$}} +define void @test_export_zeroes() #0 { + + call void @llvm.SI.export(i32 0, i32 0, i32 0, i32 0, i32 0, float 0.0, float 0.0, float 0.0, float 0.0) + call void @llvm.SI.export(i32 0, i32 0, i32 1, i32 0, i32 0, float 0.0, float 0.0, float 0.0, float 0.0) + ret void +} + +; FIXME: Should not set up registers for the unused source registers. + +; GCN-LABEL: {{^}}test_export_en_src0: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], off, off, off done{{$}} +define void @test_export_en_src0() #0 { + call void @llvm.SI.export(i32 1, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src1: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 off, [[SRC1]], off, off done{{$}} +define void @test_export_en_src1() #0 { + call void @llvm.SI.export(i32 2, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src2: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 off, off, [[SRC2]], off done{{$}} +define void @test_export_en_src2() #0 { + call void @llvm.SI.export(i32 4, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src3: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 off, off, off, [[SRC3]] done{{$}} +define void @test_export_en_src3() #0 { + call void @llvm.SI.export(i32 8, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src0_src1: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], [[SRC1]], off, off done{{$}} +define void @test_export_en_src0_src1() #0 { + call void @llvm.SI.export(i32 3, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src0_src2: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], off, [[SRC2]], off done{{$}} +define void @test_export_en_src0_src2() #0 { + call void @llvm.SI.export(i32 5, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src0_src3: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], off, off, [[SRC3]]{{$}} +; GCN: exp mrt0 [[SRC0]], off, off, [[SRC3]] done{{$}} +define void @test_export_en_src0_src3() #0 { + call void @llvm.SI.export(i32 9, i32 0, i32 0, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 9, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_en_src0_src1_src2_src3: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp mrt0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_en_src0_src1_src2_src3() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_mrt7: +; GCN-DAG: v_mov_b32_e32 [[VHALF:v[0-9]+]], 0.5 +; GCN: exp mrt7 [[VHALF]], [[VHALF]], [[VHALF]], [[VHALF]]{{$}} +; GCN: exp mrt7 [[VHALF]], [[VHALF]], [[VHALF]], [[VHALF]] done{{$}} +define void @test_export_mrt7() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 7, i32 0, float 0.5, float 0.5, float 0.5, float 0.5) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 7, i32 0, float 0.5, float 0.5, float 0.5, float 0.5) + ret void +} + +; GCN-LABEL: {{^}}test_export_z: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrtz [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp mrtz [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_z() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 8, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 8, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_null: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp null [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp null [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_null() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 9, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 9, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_reserved10: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp invalid_target_10 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp invalid_target_10 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_reserved10() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 10, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 10, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_reserved11: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp invalid_target_11 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp invalid_target_11 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_reserved11() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 11, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 11, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_pos0: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp pos0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp pos0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_pos0() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 12, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 12, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_pos3: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp pos3 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp pos3 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_pos3() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 15, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 15, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_param0: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp param0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp param0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_param0() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 32, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 32, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_param31: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp param31 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]]{{$}} +; GCN: exp param31 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done{{$}} +define void @test_export_param31() #0 { + call void @llvm.SI.export(i32 15, i32 0, i32 0, i32 63, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 0, i32 1, i32 63, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +; GCN-LABEL: {{^}}test_export_vm: +; GCN-DAG: v_mov_b32_e32 [[SRC0:v[0-9]+]], 1.0 +; GCN-DAG: v_mov_b32_e32 [[SRC1:v[0-9]+]], 2.0 +; GCN-DAG: v_mov_b32_e32 [[SRC2:v[0-9]+]], 0.5 +; GCN-DAG: v_mov_b32_e32 [[SRC3:v[0-9]+]], 4.0 +; GCN: exp mrt0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] vm{{$}} +; GCN: exp mrt0 [[SRC0]], [[SRC1]], [[SRC2]], [[SRC3]] done vm{{$}} +define void @test_export_vm() #0 { + call void @llvm.SI.export(i32 15, i32 1, i32 0, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 0, float 1.0, float 2.0, float 0.5, float 4.0) + ret void +} + +attributes #0 = { nounwind "ShaderType"="0" } diff --git a/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll b/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll index ca1faebb77e7..9e7c3c2e6201 100644 --- a/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll +++ b/test/CodeGen/AMDGPU/llvm.SI.fs.interp.ll @@ -5,10 +5,10 @@ ;GCN-LABEL: {{^}}main: ;GCN-NOT: s_wqm -;GCN: s_mov_b32 -;GCN-NEXT: v_interp_mov_f32 -;GCN: v_interp_p1_f32 -;GCN: v_interp_p2_f32 +;GCN: s_mov_b32 m0 +;GCN-DAG: v_interp_mov_f32 +;GCN-DAG: v_interp_p1_f32 +;GCN-DAG: v_interp_p2_f32 define amdgpu_ps void @main(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x i32>) { main_body: diff --git a/test/CodeGen/AMDGPU/llvm.SI.load.dword.ll b/test/CodeGen/AMDGPU/llvm.SI.load.dword.ll index d0cc00d81b4e..ee0a41f2210f 100644 --- a/test/CodeGen/AMDGPU/llvm.SI.load.dword.ll +++ b/test/CodeGen/AMDGPU/llvm.SI.load.dword.ll @@ -49,4 +49,5 @@ declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float attributes #0 = { nounwind readonly } -!0 = !{!"const", null, i32 1} +!0 = !{!"const", !1, i32 1} +!1 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll index 67c7baba3e14..9cb9f25520b8 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.format.ll @@ -32,10 +32,10 @@ main_body: ;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen ;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], 61 offset:4095 ;VI-DAG: s_movk_i32 [[OFS1:s[0-9]+]], 0x7fff -;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS1]] offset:4093 +;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS1]] offset:4093 ;SICI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, s[0:3], 0 offen -;VI: s_mov_b32 [[OFS2:s[0-9]+]], 0x8fff -;VI: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS2]] offset:1 +;VI-DAG: s_mov_b32 [[OFS2:s[0-9]+]], 0x8fff +;VI-DAG: buffer_load_format_xyzw {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], [[OFS2]] offset:1 ;CHECK: s_waitcnt define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) { main_body: diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll index 010ad276da10..5c93ae0e7867 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll @@ -112,6 +112,20 @@ main_body: ret <4 x float> %data } +; SI won't merge ds memory operations, because of the signed offset bug, so +; we only have check lines for VI. +; CHECK-LABEL: buffer_load_mmo: +; VI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 +; VI: ds_write2_b32 v{{[0-9]+}}, [[ZERO]], [[ZERO]] offset1:4 +define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, float addrspace(3)* %lds) { +entry: + store float 0.0, float addrspace(3)* %lds + %val = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 0, i1 0, i1 0) + %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4 + store float 0.0, float addrspace(3)* %tmp2 + ret float %val +} + declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #0 declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #0 declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #0 diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll index cecfcb1bfe7c..4e0f3c37f214 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.wbinvl1.vol.ll @@ -7,9 +7,11 @@ declare void @llvm.amdgcn.buffer.wbinvl1.vol() #0 ; GCN-NEXT: ; BB#0: ; CI-NEXT: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xc0,0xe1,0x00,0x00,0x00,0x00] ; VI-NEXT: buffer_wbinvl1_vol ; encoding: [0x00,0x00,0xfc,0xe0,0x00,0x00,0x00,0x00] -; GCN-NEXT: s_endpgm +; GCN: s_endpgm define void @test_buffer_wbinvl1_vol() #0 { call void @llvm.amdgcn.buffer.wbinvl1.vol() +; This used to crash in hazard recognizer + store i8 0, i8 addrspace(1)* undef, align 1 ret void } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll new file mode 100644 index 000000000000..2247485f2993 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll @@ -0,0 +1,155 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.fabs.f16(half %a) +declare i1 @llvm.amdgcn.class.f16(half %a, i32 %b) + +; GCN-LABEL: {{^}}class_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_I32:[0-9]+]] +; VI: v_cmp_class_f16_e32 vcc, v[[A_F16]], v[[B_I32]] +; GCN: v_cndmask_b32_e64 v[[R_I32:[0-9]+]] +; GCN: buffer_store_dword v[[R_I32]] +; GCN: s_endpgm +define void @class_f16( + i32 addrspace(1)* %r, + half addrspace(1)* %a, + i32 addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load i32, i32 addrspace(1)* %b + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_fabs +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; GCN: s_load_dword s[[SB_I32:[0-9]+]] +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], |v[[VA_F16]]|, s[[SB_I32]] +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, [[CMP]] +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_fabs( + i32 addrspace(1)* %r, + half %a.val, + i32 %b.val) { +entry: + %a.val.fabs = call half @llvm.fabs.f16(half %a.val) + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val.fabs, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_fneg +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; GCN: s_load_dword s[[SB_I32:[0-9]+]] +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], -v[[VA_F16]], s[[SB_I32]] +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, [[CMP]] +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_fneg( + i32 addrspace(1)* %r, + half %a.val, + i32 %b.val) { +entry: + %a.val.fneg = fsub half -0.0, %a.val + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val.fneg, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_fabs_fneg +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; GCN: s_load_dword s[[SB_I32:[0-9]+]] +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], -|v[[VA_F16]]|, s[[SB_I32]] +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, [[CMP]] +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_fabs_fneg( + i32 addrspace(1)* %r, + half %a.val, + i32 %b.val) { +entry: + %a.val.fabs = call half @llvm.fabs.f16(half %a.val) + %a.val.fabs.fneg = fsub half -0.0, %a.val.fabs + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val.fabs.fneg, i32 %b.val) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_1 +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], v[[VA_F16]], 1{{$}} +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, [[CMP]] +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_1( + i32 addrspace(1)* %r, + half %a.val) { +entry: + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 1) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_64 +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], v[[VA_F16]], 64{{$}} +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, [[CMP]] +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_64( + i32 addrspace(1)* %r, + half %a.val) { +entry: + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 64) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_full_mask +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; VI: v_mov_b32_e32 v[[MASK:[0-9]+]], 0x3ff{{$}} +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e32 vcc, v[[VA_F16]], v[[MASK]] +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, vcc +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_full_mask( + i32 addrspace(1)* %r, + half %a.val) { +entry: + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 1023) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}class_f16_nine_bit_mask +; GCN: s_load_dword s[[SA_F16:[0-9]+]] +; VI: v_mov_b32_e32 v[[MASK:[0-9]+]], 0x1ff{{$}} +; VI: v_trunc_f16_e32 v[[VA_F16:[0-9]+]], s[[SA_F16]] +; VI: v_cmp_class_f16_e32 vcc, v[[VA_F16]], v[[MASK]] +; VI: v_cndmask_b32_e64 v[[VR_I32:[0-9]+]], 0, -1, vcc +; GCN: buffer_store_dword v[[VR_I32]] +; GCN: s_endpgm +define void @class_f16_nine_bit_mask( + i32 addrspace(1)* %r, + half %a.val) { +entry: + %r.val = call i1 @llvm.amdgcn.class.f16(half %a.val, i32 511) + %r.val.sext = sext i1 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.cos.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.cos.f16.ll new file mode 100644 index 000000000000..5d9f1b824a69 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.cos.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.cos.f16(half %a) + +; GCN-LABEL: {{^}}cos_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_cos_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @cos_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.cos.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll new file mode 100644 index 000000000000..6c09aa592447 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.dispatch.id.ll @@ -0,0 +1,19 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i64 @llvm.amdgcn.dispatch.id() #1 + +; GCN-LABEL: {{^}}dispatch_id: +; GCN: .amd_kernel_code_t +; GCN: enable_sgpr_dispatch_id = 1 + +; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], s6 +; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], s7 +; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @dispatch_id(i64 addrspace(1)* %out) #0 { + %tmp0 = call i64 @llvm.amdgcn.dispatch.id() + store i64 %tmp0, i64 addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll new file mode 100644 index 000000000000..d69cd7c6068c --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fixup.f16.ll @@ -0,0 +1,129 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.div.fixup.f16(half %a, half %b, half %c) + +; GCN-LABEL: {{^}}div_fixup_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.amdgcn.div.fixup.f16(half %a.val, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; VI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x4200{{$}} +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b, + half addrspace(1)* %c) { +entry: + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.amdgcn.div.fixup.f16(half 3.0, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; VI: v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x4200{{$}} +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %c) { +entry: + %a.val = load half, half addrspace(1)* %a + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.amdgcn.div.fixup.f16(half %a.val, half 3.0, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_c +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; VI: v_mov_b32_e32 v[[C_F16:[0-9]+]], 0x4200{{$}} +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_c( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.amdgcn.div.fixup.f16(half %a.val, half %b.val, half 3.0) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_a_imm_b +; VI: v_mov_b32_e32 v[[AB_F16:[0-9]+]], 0x4200{{$}} +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[AB_F16]], v[[AB_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_a_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %c) { +entry: + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.amdgcn.div.fixup.f16(half 3.0, half 3.0, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_b_imm_c +; VI: v_mov_b32_e32 v[[BC_F16:[0-9]+]], 0x4200{{$}} +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[BC_F16]], v[[BC_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_b_imm_c( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.div.fixup.f16(half %a.val, half 3.0, half 3.0) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}div_fixup_f16_imm_a_imm_c +; VI: v_mov_b32_e32 v[[AC_F16:[0-9]+]], 0x4200{{$}} +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; VI: v_div_fixup_f16 v[[R_F16:[0-9]+]], v[[AC_F16]], v[[B_F16]], v[[AC_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @div_fixup_f16_imm_a_imm_c( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.amdgcn.div.fixup.f16(half 3.0, half %b.val, half 3.0) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll index efea3eb707a1..af934ede040a 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll @@ -77,7 +77,7 @@ define void @test_div_fmas_f64(double addrspace(1)* %out, double %a, double %b, } ; GCN-LABEL: {{^}}test_div_fmas_f32_cond_to_vcc: -; SI: v_cmp_eq_i32_e64 vcc, 0, s{{[0-9]+}} +; SI: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}} ; SI: v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} define void @test_div_fmas_f32_cond_to_vcc(float addrspace(1)* %out, float %a, float %b, float %c, i32 %i) nounwind { %cmp = icmp eq i32 %i, 0 @@ -109,8 +109,8 @@ define void @test_div_fmas_f32_imm_true_cond_to_vcc(float addrspace(1)* %out, fl ; SI-DAG: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}} ; SI-DAG: buffer_load_dword [[C:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}} -; SI-DAG: v_cmp_eq_i32_e32 [[CMP0:vcc]], 0, v{{[0-9]+}} -; SI-DAG: v_cmp_ne_i32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]], 0, s{{[0-9]+}} +; SI-DAG: v_cmp_eq_u32_e32 [[CMP0:vcc]], 0, v{{[0-9]+}} +; SI-DAG: v_cmp_ne_u32_e64 [[CMP1:s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} ; SI: s_and_b64 vcc, [[CMP0]], [[CMP1]] ; SI: v_div_fmas_f32 {{v[0-9]+}}, [[A]], [[B]], [[C]] ; SI: s_endpgm @@ -135,18 +135,18 @@ define void @test_div_fmas_f32_logical_cond_to_vcc(float addrspace(1)* %out, flo } ; GCN-LABEL: {{^}}test_div_fmas_f32_i1_phi_vcc: -; SI: v_cmp_eq_i32_e32 vcc, 0, v{{[0-9]+}} +; SI: v_cmp_eq_u32_e32 vcc, 0, v{{[0-9]+}} ; SI: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], vcc ; SI: s_xor_b64 [[SAVE]], exec, [[SAVE]] ; SI: buffer_load_dword [[LOAD:v[0-9]+]] -; SI: v_cmp_ne_i32_e32 vcc, 0, [[LOAD]] +; SI: v_cmp_ne_u32_e32 vcc, 0, [[LOAD]] ; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc ; SI: BB9_2: ; SI: s_or_b64 exec, exec, [[SAVE]] -; SI: v_cmp_ne_i32_e32 vcc, 0, v{{[0-9]+}} +; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}} ; SI: v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} ; SI: buffer_store_dword ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.ll new file mode 100644 index 000000000000..427ad5ef553d --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.ll @@ -0,0 +1,228 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0 +declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0 +declare float @llvm.fabs.f32(float) #0 + +; GCN-LABEL: {{^}}v_fcmp_f32_oeq_with_fabs: +; GCN: v_cmp_eq_f32_e64 {{s\[[0-9]+:[0-9]+\]}}, |{{v[0-9]+}}|, {{s[0-9]+}} +define void @v_fcmp_f32_oeq_with_fabs(i64 addrspace(1)* %out, float %src, float %a) { + %temp = call float @llvm.fabs.f32(float %a) + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float %temp, i32 1) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_oeq_both_operands_with_fabs: +; GCN: v_cmp_eq_f32_e64 {{s\[[0-9]+:[0-9]+\]}}, |{{v[0-9]+}}|, |{{s[0-9]+}}| +define void @v_fcmp_f32_oeq_both_operands_with_fabs(i64 addrspace(1)* %out, float %src, float %a) { + %temp = call float @llvm.fabs.f32(float %a) + %src_input = call float @llvm.fabs.f32(float %src) + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src_input, float %temp, i32 1) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp: +; GCN-NOT: v_cmp_eq_f32_e64 +define void @v_fcmp(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 -1) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_oeq: +; GCN: v_cmp_eq_f32_e64 +define void @v_fcmp_f32_oeq(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 1) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_one: +; GCN: v_cmp_neq_f32_e64 +define void @v_fcmp_f32_one(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 6) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_ogt: +; GCN: v_cmp_gt_f32_e64 +define void @v_fcmp_f32_ogt(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 2) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_oge: +; GCN: v_cmp_ge_f32_e64 +define void @v_fcmp_f32_oge(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 3) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_olt: +; GCN: v_cmp_lt_f32_e64 +define void @v_fcmp_f32_olt(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 4) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_ole: +; GCN: v_cmp_le_f32_e64 +define void @v_fcmp_f32_ole(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 5) + store i64 %result, i64 addrspace(1)* %out + ret void +} + + +; GCN-LABEL: {{^}}v_fcmp_f32_ueq: +; GCN: v_cmp_nlg_f32_e64 +define void @v_fcmp_f32_ueq(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 9) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_une: +; GCN: v_cmp_neq_f32_e64 +define void @v_fcmp_f32_une(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 14) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_ugt: +; GCN: v_cmp_nle_f32_e64 +define void @v_fcmp_f32_ugt(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 10) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_uge: +; GCN: v_cmp_nlt_f32_e64 +define void @v_fcmp_f32_uge(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 11) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_ult: +; GCN: v_cmp_nge_f32_e64 +define void @v_fcmp_f32_ult(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 12) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f32_ule: +; GCN: v_cmp_ngt_f32_e64 +define void @v_fcmp_f32_ule(i64 addrspace(1)* %out, float %src) { + %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 13) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_oeq: +; GCN: v_cmp_eq_f64_e64 +define void @v_fcmp_f64_oeq(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 1) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_one: +; GCN: v_cmp_neq_f64_e64 +define void @v_fcmp_f64_one(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 6) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ogt: +; GCN: v_cmp_gt_f64_e64 +define void @v_fcmp_f64_ogt(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 2) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_oge: +; GCN: v_cmp_ge_f64_e64 +define void @v_fcmp_f64_oge(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 3) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_olt: +; GCN: v_cmp_lt_f64_e64 +define void @v_fcmp_f64_olt(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 4) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ole: +; GCN: v_cmp_le_f64_e64 +define void @v_fcmp_f64_ole(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 5) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ueq: +; GCN: v_cmp_nlg_f64_e64 +define void @v_fcmp_f64_ueq(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 9) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_une: +; GCN: v_cmp_neq_f64_e64 +define void @v_fcmp_f64_une(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 14) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ugt: +; GCN: v_cmp_nle_f64_e64 +define void @v_fcmp_f64_ugt(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 10) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_uge: +; GCN: v_cmp_nlt_f64_e64 +define void @v_fcmp_f64_uge(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 11) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ult: +; GCN: v_cmp_nge_f64_e64 +define void @v_fcmp_f64_ult(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 12) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_fcmp_f64_ule: +; GCN: v_cmp_ngt_f64_e64 +define void @v_fcmp_f64_ule(i64 addrspace(1)* %out, double %src) { + %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 13) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind readnone convergent } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.fmul.legacy.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.fmul.legacy.ll new file mode 100644 index 000000000000..d5c1c0a0969b --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.fmul.legacy.ll @@ -0,0 +1,54 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + + +; GCN-LABEL: {{^}}test_mul_legacy_f32: +; GCN: v_mul_legacy_f32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}} +define void @test_mul_legacy_f32(float addrspace(1)* %out, float %a, float %b) #0 { + %result = call float @llvm.amdgcn.fmul.legacy(float %a, float %b) + store float %result, float addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}test_mul_legacy_undef0_f32: +; GCN: v_mul_legacy_f32_e32 +define void @test_mul_legacy_undef0_f32(float addrspace(1)* %out, float %a) #0 { + %result = call float @llvm.amdgcn.fmul.legacy(float undef, float %a) + store float %result, float addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}test_mul_legacy_undef1_f32: +; GCN: v_mul_legacy_f32_e32 +define void @test_mul_legacy_undef1_f32(float addrspace(1)* %out, float %a) #0 { + %result = call float @llvm.amdgcn.fmul.legacy(float %a, float undef) + store float %result, float addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}test_mul_legacy_fabs_f32: +; GCN: v_mul_legacy_f32_e64 v{{[0-9]+}}, |v{{[0-9]+}}|, |s{{[0-9]+}}| +define void @test_mul_legacy_fabs_f32(float addrspace(1)* %out, float %a, float %b) #0 { + %a.fabs = call float @llvm.fabs.f32(float %a) + %b.fabs = call float @llvm.fabs.f32(float %b) + %result = call float @llvm.amdgcn.fmul.legacy(float %a.fabs, float %b.fabs) + store float %result, float addrspace(1)* %out, align 4 + ret void +} + +; TODO: Should match mac_legacy/mad_legacy +; GCN-LABEL: {{^}}test_mad_legacy_f32: +; GCN: v_mul_legacy_f32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_add_f32_e32 +define void @test_mad_legacy_f32(float addrspace(1)* %out, float %a, float %b, float %c) #0 { + %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b) + %add = fadd float %mul, %c + store float %add, float addrspace(1)* %out, align 4 + ret void +} + +declare float @llvm.fabs.f32(float) #1 +declare float @llvm.amdgcn.fmul.legacy(float, float) #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.fract.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.fract.f16.ll new file mode 100644 index 000000000000..c1af0ecb3edc --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.fract.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.fract.f16(half %a) + +; GCN-LABEL: {{^}}fract_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_fract_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fract_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.fract.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll new file mode 100644 index 000000000000..51a1e0776728 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare i16 @llvm.amdgcn.frexp.exp.i16.f16(half %a) + +; GCN-LABEL: {{^}}frexp_exp_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_frexp_exp_i16_f16_e32 v[[R_I16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_I16]] +define void @frexp_exp_f16( + i16 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call i16 @llvm.amdgcn.frexp.exp.i16.f16(half %a.val) + store i16 %r.val, i16 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}frexp_exp_f16_sext +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_frexp_exp_i16_f16_e32 v[[R_I16:[0-9]+]], v[[A_F16]] +; VI: v_bfe_i32 v[[R_I32:[0-9]+]], v[[R_I16]], 0, 16{{$}} +; GCN: buffer_store_dword v[[R_I32]] +define void @frexp_exp_f16_sext( + i32 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call i16 @llvm.amdgcn.frexp.exp.i16.f16(half %a.val) + %r.val.sext = sext i16 %r.val to i32 + store i32 %r.val.sext, i32 addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}frexp_exp_f16_zext +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_frexp_exp_i16_f16_e32 v[[R_I16:[0-9]+]], v[[A_F16]] +; VI: v_and_b32_e32 v[[R_I32:[0-9]+]], 0xffff, v[[R_I16]] +; GCN: buffer_store_dword v[[R_I32]] +define void @frexp_exp_f16_zext( + i32 addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call i16 @llvm.amdgcn.frexp.exp.i16.f16(half %a.val) + %r.val.zext = zext i16 %r.val to i32 + store i32 %r.val.zext, i32 addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll index 728a6b5cf26b..94e92f6f1690 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll @@ -3,13 +3,13 @@ declare float @llvm.fabs.f32(float) #0 declare double @llvm.fabs.f64(double) #0 -declare i32 @llvm.amdgcn.frexp.exp.f32(float) #0 -declare i32 @llvm.amdgcn.frexp.exp.f64(double) #0 +declare i32 @llvm.amdgcn.frexp.exp.i32.f32(float) #0 +declare i32 @llvm.amdgcn.frexp.exp.i32.f64(double) #0 ; GCN-LABEL: {{^}}s_test_frexp_exp_f32: ; GCN: v_frexp_exp_i32_f32_e32 {{v[0-9]+}}, {{s[0-9]+}} define void @s_test_frexp_exp_f32(i32 addrspace(1)* %out, float %src) #1 { - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f32(float %src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f32(float %src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } @@ -18,7 +18,7 @@ define void @s_test_frexp_exp_f32(i32 addrspace(1)* %out, float %src) #1 { ; GCN: v_frexp_exp_i32_f32_e64 {{v[0-9]+}}, |{{s[0-9]+}}| define void @s_test_fabs_frexp_exp_f32(i32 addrspace(1)* %out, float %src) #1 { %fabs.src = call float @llvm.fabs.f32(float %src) - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f32(float %fabs.src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f32(float %fabs.src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } @@ -28,7 +28,7 @@ define void @s_test_fabs_frexp_exp_f32(i32 addrspace(1)* %out, float %src) #1 { define void @s_test_fneg_fabs_frexp_exp_f32(i32 addrspace(1)* %out, float %src) #1 { %fabs.src = call float @llvm.fabs.f32(float %src) %fneg.fabs.src = fsub float -0.0, %fabs.src - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f32(float %fneg.fabs.src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f32(float %fneg.fabs.src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } @@ -36,7 +36,7 @@ define void @s_test_fneg_fabs_frexp_exp_f32(i32 addrspace(1)* %out, float %src) ; GCN-LABEL: {{^}}s_test_frexp_exp_f64: ; GCN: v_frexp_exp_i32_f64_e32 {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} define void @s_test_frexp_exp_f64(i32 addrspace(1)* %out, double %src) #1 { - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f64(double %src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f64(double %src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } @@ -45,7 +45,7 @@ define void @s_test_frexp_exp_f64(i32 addrspace(1)* %out, double %src) #1 { ; GCN: v_frexp_exp_i32_f64_e64 {{v[0-9]+}}, |{{s\[[0-9]+:[0-9]+\]}}| define void @s_test_fabs_frexp_exp_f64(i32 addrspace(1)* %out, double %src) #1 { %fabs.src = call double @llvm.fabs.f64(double %src) - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f64(double %fabs.src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f64(double %fabs.src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } @@ -55,7 +55,7 @@ define void @s_test_fabs_frexp_exp_f64(i32 addrspace(1)* %out, double %src) #1 { define void @s_test_fneg_fabs_frexp_exp_f64(i32 addrspace(1)* %out, double %src) #1 { %fabs.src = call double @llvm.fabs.f64(double %src) %fneg.fabs.src = fsub double -0.0, %fabs.src - %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.f64(double %fneg.fabs.src) + %frexp.exp = call i32 @llvm.amdgcn.frexp.exp.i32.f64(double %fneg.fabs.src) store i32 %frexp.exp, i32 addrspace(1)* %out ret void } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.mant.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.mant.f16.ll new file mode 100644 index 000000000000..4b33549537af --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.mant.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.frexp.mant.f16(half %a) + +; GCN-LABEL: {{^}}frexp_mant_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_frexp_mant_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @frexp_mant_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.frexp.mant.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll new file mode 100644 index 000000000000..6d0457bc6489 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll @@ -0,0 +1,172 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0 +declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0 + +; GCN-LABEL: {{^}}v_icmp_i32_eq: +; GCN: v_cmp_eq_u32_e64 +define void @v_icmp_i32_eq(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 32) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp: +; GCN-NOT: v_cmp_eq_u32_e64 +define void @v_icmp(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 30) + store i64 %result, i64 addrspace(1)* %out + ret void +} +; GCN-LABEL: {{^}}v_icmp_i32_ne: +; GCN: v_cmp_ne_u32_e64 +define void @v_icmp_i32_ne(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 33) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u32_ugt: +; GCN: v_cmp_gt_u32_e64 +define void @v_icmp_u32_ugt(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 34) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u32_uge: +; GCN: v_cmp_ge_u32_e64 +define void @v_icmp_u32_uge(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 35) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u32_ult: +; GCN: v_cmp_lt_u32_e64 +define void @v_icmp_u32_ult(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 36) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u32_ule: +; GCN: v_cmp_le_u32_e64 +define void @v_icmp_u32_ule(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 37) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i32_sgt: +; GCN: v_cmp_gt_i32_e64 +define void @v_icmp_i32_sgt(i64 addrspace(1)* %out, i32 %src) #1 { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 38) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i32_sge: +; GCN: v_cmp_ge_i32_e64 +define void @v_icmp_i32_sge(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 39) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i32_slt: +; GCN: v_cmp_lt_i32_e64 +define void @v_icmp_i32_slt(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 40) + store i64 %result, i64 addrspace(1)* %out + ret void +} +; GCN-LABEL: {{^}}v_icmp_i32_sle: +; GCN: v_cmp_le_i32_e64 +define void @v_icmp_i32_sle(i64 addrspace(1)* %out, i32 %src) { + %result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 41) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i64_eq: +; GCN: v_cmp_eq_u64_e64 +define void @v_icmp_i64_eq(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 32) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i64_ne: +; GCN: v_cmp_ne_u64_e64 +define void @v_icmp_i64_ne(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 33) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u64_ugt: +; GCN: v_cmp_gt_u64_e64 +define void @v_icmp_u64_ugt(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 34) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u64_uge: +; GCN: v_cmp_ge_u64_e64 +define void @v_icmp_u64_uge(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 35) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u64_ult: +; GCN: v_cmp_lt_u64_e64 +define void @v_icmp_u64_ult(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 36) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_u64_ule: +; GCN: v_cmp_le_u64_e64 +define void @v_icmp_u64_ule(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 37) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i64_sgt: +; GCN: v_cmp_gt_i64_e64 +define void @v_icmp_i64_sgt(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 38) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i64_sge: +; GCN: v_cmp_ge_i64_e64 +define void @v_icmp_i64_sge(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 39) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_icmp_i64_slt: +; GCN: v_cmp_lt_i64_e64 +define void @v_icmp_i64_slt(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 40) + store i64 %result, i64 addrspace(1)* %out + ret void +} +; GCN-LABEL: {{^}}v_icmp_i64_sle: +; GCN: v_cmp_le_i64_e64 +define void @v_icmp_i64_sle(i64 addrspace(1)* %out, i64 %src) { + %result = call i64 @llvm.amdgcn.icmp.i64(i64 %src, i64 100, i32 41) + store i64 %result, i64 addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind readnone convergent } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.ll new file mode 100644 index 000000000000..a65f422742c9 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.ll @@ -0,0 +1,383 @@ +; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s + +; GCN-LABEL: {{^}}gather4_v2: +; GCN: image_gather4 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_v2(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.v4f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4: +; GCN: image_gather4 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_cl: +; GCN: image_gather4_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_l: +; GCN: image_gather4_l {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.l.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b: +; GCN: image_gather4_b {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b_cl: +; GCN: image_gather4_b_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b_cl_v8: +; GCN: image_gather4_b_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b_cl_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_lz_v2: +; GCN: image_gather4_lz {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_lz_v2(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.lz.v4f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_lz: +; GCN: image_gather4_lz {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.lz.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + + + +; GCN-LABEL: {{^}}gather4_o: +; GCN: image_gather4_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_cl_o: +; GCN: image_gather4_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_cl_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_cl_o_v8: +; GCN: image_gather4_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_cl_o_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_l_o: +; GCN: image_gather4_l_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_l_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.l.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_l_o_v8: +; GCN: image_gather4_l_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_l_o_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.l.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b_o: +; GCN: image_gather4_b_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b_o_v8: +; GCN: image_gather4_b_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b_o_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_b_cl_o: +; GCN: image_gather4_b_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_b_cl_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_lz_o: +; GCN: image_gather4_lz_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_lz_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + + +; GCN-LABEL: {{^}}gather4_c: +; GCN: image_gather4_c {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_cl: +; GCN: image_gather4_c_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_cl_v8: +; GCN: image_gather4_c_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_cl_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_l: +; GCN: image_gather4_c_l {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.l.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_l_v8: +; GCN: image_gather4_c_l {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_l_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.l.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_b: +; GCN: image_gather4_c_b {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.b.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_b_v8: +; GCN: image_gather4_c_b {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_b_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.b.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_b_cl: +; GCN: image_gather4_c_b_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_lz: +; GCN: image_gather4_c_lz {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + + +; GCN-LABEL: {{^}}gather4_c_o: +; GCN: image_gather4_c_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_o_v8: +; GCN: image_gather4_c_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_o_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_cl_o: +; GCN: image_gather4_c_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_cl_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_l_o: +; GCN: image_gather4_c_l_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_l_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_b_o: +; GCN: image_gather4_c_b_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_b_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_b_cl_o: +; GCN: image_gather4_c_b_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_b_cl_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_lz_o: +; GCN: image_gather4_c_lz_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_lz_o(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_c_lz_o_v8: +; GCN: image_gather4_c_lz_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_c_lz_o_v8(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.v4f32.v8f32.v8i32(<8 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_f32: +; GCN: image_gather4 {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 da +define void @gather4_f32(float addrspace(1)* %out) { +main_body: + %r = call float @llvm.amdgcn.image.gather4.f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 1) + store float %r, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}gather4_v2f32: +; GCN: image_gather4 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 da +define void @gather4_v2f32(<2 x float> addrspace(1)* %out) { +main_body: + %r = call <2 x float> @llvm.amdgcn.image.gather4.v2f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 0, i1 0, i1 0, i1 0, i1 1) + store <2 x float> %r, <2 x float> addrspace(1)* %out + ret void +} + +declare <4 x float> @llvm.amdgcn.image.gather4.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.l.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.lz.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.lz.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.gather4.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.l.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.l.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.gather4.c.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.l.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.l.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.b.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.b.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.gather4.c.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.v4f32.v8f32.v8i32(<8 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare float @llvm.amdgcn.image.gather4.f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <2 x float> @llvm.amdgcn.image.gather4.v2f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll new file mode 100644 index 000000000000..ef810a330017 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.getlod.ll @@ -0,0 +1,37 @@ +; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s + +; GCN-LABEL: {{^}}getlod: +; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da +define void @getlod(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.f32.v8i32(float undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}getlod_v2: +; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da +define void @getlod_v2(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}getlod_v4: +; GCN: image_get_lod {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf da +define void @getlod_v4(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.getlod.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 1) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + + +declare <4 x float> @llvm.amdgcn.image.getlod.v4f32.f32.v8i32(float, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.getlod.v4f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.getlod.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll index f0d23b93119d..69c43ca3070a 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.ll @@ -1,12 +1,12 @@ ;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck %s -;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s +;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=CHECK,VI %s ;CHECK-LABEL: {{^}}image_load_v4i32: ;CHECK: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm ;CHECK: s_waitcnt vmcnt(0) define amdgpu_ps <4 x float> @image_load_v4i32(<8 x i32> inreg %rsrc, <4 x i32> %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret <4 x float> %tex } @@ -15,7 +15,7 @@ main_body: ;CHECK: s_waitcnt vmcnt(0) define amdgpu_ps <4 x float> @image_load_v2i32(<8 x i32> inreg %rsrc, <2 x i32> %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.v2i32(<2 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret <4 x float> %tex } @@ -24,7 +24,7 @@ main_body: ;CHECK: s_waitcnt vmcnt(0) define amdgpu_ps <4 x float> @image_load_i32(<8 x i32> inreg %rsrc, i32 %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.i32(i32 %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32 %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret <4 x float> %tex } @@ -33,7 +33,7 @@ main_body: ;CHECK: s_waitcnt vmcnt(0) define amdgpu_ps <4 x float> @image_load_mip(<8 x i32> inreg %rsrc, <4 x i32> %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.mip.v4i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.mip.v4f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret <4 x float> %tex } @@ -42,17 +42,36 @@ main_body: ;CHECK: s_waitcnt vmcnt(0) define amdgpu_ps float @image_load_1(<8 x i32> inreg %rsrc, <4 x i32> %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) %elt = extractelement <4 x float> %tex, i32 0 ; Only first component used, test that dmask etc. is changed accordingly ret float %elt } +;CHECK-LABEL: {{^}}image_load_f32_v2i32: +;CHECK: image_load {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 unorm +;CHECK: s_waitcnt vmcnt(0) +define amdgpu_ps float @image_load_f32_v2i32(<8 x i32> inreg %rsrc, <2 x i32> %c) { +main_body: + %tex = call float @llvm.amdgcn.image.load.f32.v2i32.v8i32(<2 x i32> %c, <8 x i32> %rsrc, i32 1, i1 0, i1 0, i1 0, i1 0) + ret float %tex +} + +;CHECK-LABEL: {{^}}image_load_v2f32_v4i32: +;CHECK: image_load {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 unorm +;CHECK: s_waitcnt vmcnt(0) +define amdgpu_ps <2 x float> @image_load_v2f32_v4i32(<8 x i32> inreg %rsrc, <4 x i32> %c) { +main_body: + %tex = call <2 x float> @llvm.amdgcn.image.load.v2f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 3, i1 0, i1 0, i1 0, i1 0) + ret <2 x float> %tex +} + + ;CHECK-LABEL: {{^}}image_store_v4i32: ;CHECK: image_store v[0:3], v[4:7], s[0:7] dmask:0xf unorm define amdgpu_ps void @image_store_v4i32(<8 x i32> inreg %rsrc, <4 x float> %data, <4 x i32> %coords) { main_body: - call void @llvm.amdgcn.image.store.v4i32(<4 x float> %data, <4 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float> %data, <4 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret void } @@ -60,7 +79,7 @@ main_body: ;CHECK: image_store v[0:3], v[4:5], s[0:7] dmask:0xf unorm define amdgpu_ps void @image_store_v2i32(<8 x i32> inreg %rsrc, <4 x float> %data, <2 x i32> %coords) { main_body: - call void @llvm.amdgcn.image.store.v2i32(<4 x float> %data, <2 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.v2i32.v8i32(<4 x float> %data, <2 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret void } @@ -68,7 +87,23 @@ main_body: ;CHECK: image_store v[0:3], v4, s[0:7] dmask:0xf unorm define amdgpu_ps void @image_store_i32(<8 x i32> inreg %rsrc, <4 x float> %data, i32 %coords) { main_body: - call void @llvm.amdgcn.image.store.i32(<4 x float> %data, i32 %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %data, i32 %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + ret void +} + +;CHECK-LABEL: {{^}}image_store_f32_i32: +;CHECK: image_store {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 unorm +define amdgpu_ps void @image_store_f32_i32(<8 x i32> inreg %rsrc, float %data, i32 %coords) { +main_body: + call void @llvm.amdgcn.image.store.f32.i32.v8i32(float %data, i32 %coords, <8 x i32> %rsrc, i32 1, i1 0, i1 0, i1 0, i1 0) + ret void +} + +;CHECK-LABEL: {{^}}image_store_v2f32_v4i32: +;CHECK: image_store {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 unorm +define amdgpu_ps void @image_store_v2f32_v4i32(<8 x i32> inreg %rsrc, <2 x float> %data, <4 x i32> %coords) { +main_body: + call void @llvm.amdgcn.image.store.v2f32.v4i32.v8i32(<2 x float> %data, <4 x i32> %coords, <8 x i32> %rsrc, i32 3, i1 0, i1 0, i1 0, i1 0) ret void } @@ -76,7 +111,20 @@ main_body: ;CHECK: image_store_mip v[0:3], v[4:7], s[0:7] dmask:0xf unorm define amdgpu_ps void @image_store_mip(<8 x i32> inreg %rsrc, <4 x float> %data, <4 x i32> %coords) { main_body: - call void @llvm.amdgcn.image.store.mip.v4i32(<4 x float> %data, <4 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.mip.v4f32.v4i32.v8i32(<4 x float> %data, <4 x i32> %coords, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + ret void +} + +;CHECK-LABEL: {{^}}getresinfo: +;CHECK: image_get_resinfo {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define amdgpu_ps void @getresinfo() { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32 undef, <8 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0) + %r0 = extractelement <4 x float> %r, i32 0 + %r1 = extractelement <4 x float> %r, i32 1 + %r2 = extractelement <4 x float> %r, i32 2 + %r3 = extractelement <4 x float> %r, i32 3 + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %r0, float %r1, float %r2, float %r3) ret void } @@ -90,21 +138,45 @@ main_body: ;CHECK: image_store v[0:3], v4, s[16:23] dmask:0xf unorm define amdgpu_ps void @image_store_wait(<8 x i32> inreg, <8 x i32> inreg, <8 x i32> inreg, <4 x float>, i32) { main_body: - call void @llvm.amdgcn.image.store.i32(<4 x float> %3, i32 %4, <8 x i32> %0, i32 15, i1 0, i1 0, i1 0, i1 0) - %data = call <4 x float> @llvm.amdgcn.image.load.i32(i32 %4, <8 x i32> %1, i32 15, i1 0, i1 0, i1 0, i1 0) - call void @llvm.amdgcn.image.store.i32(<4 x float> %data, i32 %4, <8 x i32> %2, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %3, i32 %4, <8 x i32> %0, i32 15, i1 0, i1 0, i1 0, i1 0) + %data = call <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32 %4, <8 x i32> %1, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %data, i32 %4, <8 x i32> %2, i32 15, i1 0, i1 0, i1 0, i1 0) ret void } -declare void @llvm.amdgcn.image.store.i32(<4 x float>, i32, <8 x i32>, i32, i1, i1, i1, i1) #0 -declare void @llvm.amdgcn.image.store.v2i32(<4 x float>, <2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 -declare void @llvm.amdgcn.image.store.v4i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 -declare void @llvm.amdgcn.image.store.mip.v4i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 +; SI won't merge ds memory operations, because of the signed offset bug, so +; we only have check lines for VI. +; VI-LABEL: image_load_mmo +; VI: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 +; VI: ds_write2_b32 v{{[0-9]+}}, [[ZERO]], [[ZERO]] offset1:4 +define amdgpu_ps void @image_load_mmo(float addrspace(3)* %lds, <2 x i32> %c, <8 x i32> inreg %rsrc) { + store float 0.0, float addrspace(3)* %lds + %tex = call float @llvm.amdgcn.image.load.f32.v2i32.v8i32(<2 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4 + store float 0.0, float addrspace(3)* %tmp2 + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %tex, float %tex, float %tex, float %tex) + ret void +} + +declare float @llvm.amdgcn.image.load.f32.v2i32.v8i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare <2 x float> @llvm.amdgcn.image.load.v2f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare void @llvm.amdgcn.image.store.f32.i32.v8i32(float, i32, <8 x i32>, i32, i1, i1, i1, i1) #0 +declare void @llvm.amdgcn.image.store.v2f32.v4i32.v8i32(<2 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 + + +declare void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float>, i32, <8 x i32>, i32, i1, i1, i1, i1) #0 +declare void @llvm.amdgcn.image.store.v4f32.v2i32.v8i32(<4 x float>, <2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 +declare void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 +declare void @llvm.amdgcn.image.store.mip.v4f32.v4i32.v8i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare <4 x float> @llvm.amdgcn.image.load.mip.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 + +declare <4 x float> @llvm.amdgcn.image.getresinfo.v4f32.i32.v8i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #0 -declare <4 x float> @llvm.amdgcn.image.load.i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #1 -declare <4 x float> @llvm.amdgcn.image.load.v2i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 -declare <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 -declare <4 x float> @llvm.amdgcn.image.load.mip.v4i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) attributes #0 = { nounwind } attributes #1 = { nounwind readonly } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.ll new file mode 100644 index 000000000000..752ec2d42fac --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.ll @@ -0,0 +1,227 @@ +; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s + +; GCN-LABEL: {{^}}sample: +; GCN: image_sample {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cl: +; GCN: image_sample_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_d: +; GCN: image_sample_d {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_d(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.d.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_d_cl: +; GCN: image_sample_d_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_d_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.d.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_l: +; GCN: image_sample_l {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.l.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_b: +; GCN: image_sample_b {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.b.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_b_cl: +; GCN: image_sample_b_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.b.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_lz: +; GCN: image_sample_lz {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.lz.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cd: +; GCN: image_sample_cd {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cd(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cd.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cd_cl: +; GCN: image_sample_cd_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cd_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c: +; GCN: image_sample_c {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cl: +; GCN: image_sample_c_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_d: +; GCN: image_sample_c_d {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_d(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.d.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_d_cl: +; GCN: image_sample_c_d_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_d_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_l: +; GCN: image_sample_c_l {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.l.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_b: +; GCN: image_sample_c_b {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.b.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_b_cl: +; GCN: image_sample_c_b_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_lz: +; GCN: image_sample_c_lz {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.lz.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cd: +; GCN: image_sample_c_cd {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cd(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cd.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cd_cl: +; GCN: image_sample_c_cd_cl {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cd_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_f32: +; GCN: image_sample {{v[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x1 +define void @sample_f32(float addrspace(1)* %out) { +main_body: + %r = call float @llvm.amdgcn.image.sample.f32.v2f32.v8i32(<2 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 1, i1 0, i1 0, i1 0, i1 0, i1 0) + store float %r, float addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_v2f32: +; GCN: image_sample {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0x3 +define void @sample_v2f32(<2 x float> addrspace(1)* %out) { +main_body: + %r = call <2 x float> @llvm.amdgcn.image.sample.v2f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 3, i1 0, i1 0, i1 0, i1 0, i1 0) + store <2 x float> %r, <2 x float> addrspace(1)* %out + ret void +} + +declare <4 x float> @llvm.amdgcn.image.sample.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.d.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.d.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.l.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.b.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.b.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.lz.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cd.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.sample.c.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.d.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.l.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.b.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.lz.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cd.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare float @llvm.amdgcn.image.sample.f32.v2f32.v8i32(<2 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <2 x float> @llvm.amdgcn.image.sample.v2f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.o.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.o.ll new file mode 100644 index 000000000000..d10fd0824692 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.o.ll @@ -0,0 +1,208 @@ +; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s + +; GCN-LABEL: {{^}}sample: +; GCN: image_sample_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cl: +; GCN: image_sample_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_d: +; GCN: image_sample_d_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_d(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.d.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_d_cl: +; GCN: image_sample_d_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_d_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_l: +; GCN: image_sample_l_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.l.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_b: +; GCN: image_sample_b_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.b.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_b_cl: +; GCN: image_sample_b_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_lz: +; GCN: image_sample_lz_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.lz.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cd: +; GCN: image_sample_cd_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cd(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cd.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_cd_cl: +; GCN: image_sample_cd_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_cd_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c: +; GCN: image_sample_c_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cl: +; GCN: image_sample_c_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_d: +; GCN: image_sample_c_d_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_d(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_d_cl: +; GCN: image_sample_c_d_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_d_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_l: +; GCN: image_sample_c_l_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_l(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_b: +; GCN: image_sample_c_b_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_b(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_b_cl: +; GCN: image_sample_c_b_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_b_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_lz: +; GCN: image_sample_c_lz_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_lz(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cd: +; GCN: image_sample_c_cd_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cd(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}sample_c_cd_cl: +; GCN: image_sample_c_cd_cl_o {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}} dmask:0xf +define void @sample_c_cd_cl(<4 x float> addrspace(1)* %out) { +main_body: + %r = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.v4f32.v4f32.v8i32(<4 x float> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0, i1 0) + store <4 x float> %r, <4 x float> addrspace(1)* %out + ret void +} + + +declare <4 x float> @llvm.amdgcn.image.sample.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.d.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.l.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.b.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.lz.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cd.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + +declare <4 x float> @llvm.amdgcn.image.sample.c.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.v4f32.v4f32.v8i32(<4 x float>, <8 x i32>, <4 x i32>, i32, i1, i1, i1, i1, i1) #0 + + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.interp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.interp.ll index 911d7d9b74d9..9ba5c69a9a24 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.interp.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.interp.ll @@ -1,28 +1,182 @@ ;RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=GCN %s -;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=GCN %s +;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefixes=GCN,VI %s -;GCN-LABEL: {{^}}v_interp: -;GCN-NOT: s_wqm -;GCN: s_mov_b32 m0, s{{[0-9]+}} -;GCN: v_interp_p1_f32 -;GCN: v_interp_p2_f32 -define amdgpu_ps void @v_interp(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x i32>) { +; GCN-LABEL: {{^}}v_interp: +; GCN-NOT: s_wqm +; GCN: s_mov_b32 m0, s{{[0-9]+}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}} +define amdgpu_ps void @v_interp(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg, <2 x float>) { main_body: - %i = extractelement <2 x i32> %4, i32 0 - %j = extractelement <2 x i32> %4, i32 1 - %p0_0 = call float @llvm.amdgcn.interp.p1(i32 %i, i32 0, i32 0, i32 %3) - %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, i32 %j, i32 0, i32 0, i32 %3) - %p0_1 = call float @llvm.amdgcn.interp.p1(i32 %i, i32 1, i32 0, i32 %3) - %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, i32 %j, i32 1, i32 0, i32 %3) - call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %p0_0, float %p0_0, float %p1_1, float %p1_1) + %i = extractelement <2 x float> %4, i32 0 + %j = extractelement <2 x float> %4, i32 1 + %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 %3) + %p1_0 = call float @llvm.amdgcn.interp.p2(float %p0_0, float %j, i32 0, i32 0, i32 %3) + %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 %3) + %p1_1 = call float @llvm.amdgcn.interp.p2(float %p0_1, float %j, i32 1, i32 0, i32 %3) + %const = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %3) + %w = fadd float %p1_1, %const + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %p0_0, float %p0_0, float %p1_1, float %w) + ret void +} + +; GCN-LABEL: {{^}}v_interp_p1: +; GCN: s_movk_i32 m0, 0x100 +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.z{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.w{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} + +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr1.x{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr2.y{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr3.z{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr4.w{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr63.w{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr64.w{{$}} +; GCN-DAG: v_interp_p1_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr64.x{{$}} +define amdgpu_ps void @v_interp_p1(float %i) { + %p0_0 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 0, i32 256) + %p0_1 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 0, i32 256) + %p0_2 = call float @llvm.amdgcn.interp.p1(float %i, i32 2, i32 0, i32 256) + %p0_3 = call float @llvm.amdgcn.interp.p1(float %i, i32 3, i32 0, i32 256) + %p0_4 = call float @llvm.amdgcn.interp.p1(float %i, i32 4, i32 0, i32 256) + %p0_5 = call float @llvm.amdgcn.interp.p1(float %i, i32 0, i32 1, i32 256) + %p0_6 = call float @llvm.amdgcn.interp.p1(float %i, i32 1, i32 2, i32 256) + %p0_7 = call float @llvm.amdgcn.interp.p1(float %i, i32 2, i32 3, i32 256) + %p0_8 = call float @llvm.amdgcn.interp.p1(float %i, i32 3, i32 4, i32 256) + %p0_9 = call float @llvm.amdgcn.interp.p1(float %i, i32 3, i32 63, i32 256) + %p0_10 = call float @llvm.amdgcn.interp.p1(float %i, i32 3, i32 64, i32 256) + %p0_11 = call float @llvm.amdgcn.interp.p1(float %i, i32 4, i32 64, i32 256) + + store volatile float %p0_0, float addrspace(1)* undef + store volatile float %p0_1, float addrspace(1)* undef + store volatile float %p0_2, float addrspace(1)* undef + store volatile float %p0_3, float addrspace(1)* undef + store volatile float %p0_4, float addrspace(1)* undef + store volatile float %p0_5, float addrspace(1)* undef + store volatile float %p0_6, float addrspace(1)* undef + store volatile float %p0_7, float addrspace(1)* undef + store volatile float %p0_8, float addrspace(1)* undef + store volatile float %p0_9, float addrspace(1)* undef + store volatile float %p0_10, float addrspace(1)* undef + store volatile float %p0_11, float addrspace(1)* undef + ret void +} + +; GCN-LABEL: {{^}}v_interp_p2: +; GCN: s_movk_i32 m0, 0x100 +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.y{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.z{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.w{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr0.x{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr63.x{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr64.x{{$}} +; GCN-DAG: v_interp_p2_f32 v{{[0-9]+}}, v{{[0-9]+}}, attr64.x{{$}} +define amdgpu_ps void @v_interp_p2(float %x, float %j) { + %p2_0 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 0, i32 0, i32 256) + %p2_1 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 1, i32 0, i32 256) + %p2_2 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 2, i32 0, i32 256) + %p2_3 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 3, i32 0, i32 256) + %p2_4 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 4, i32 0, i32 256) + + %p2_5 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 0, i32 1, i32 256) + %p2_6 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 0, i32 63, i32 256) + %p2_7 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 0, i32 64, i32 256) + %p2_8 = call float @llvm.amdgcn.interp.p2(float %x, float %j, i32 4, i32 64, i32 256) + + store volatile float %p2_0, float addrspace(1)* undef + store volatile float %p2_1, float addrspace(1)* undef + store volatile float %p2_2, float addrspace(1)* undef + store volatile float %p2_3, float addrspace(1)* undef + store volatile float %p2_4, float addrspace(1)* undef + store volatile float %p2_5, float addrspace(1)* undef + store volatile float %p2_6, float addrspace(1)* undef + store volatile float %p2_7, float addrspace(1)* undef + store volatile float %p2_8, float addrspace(1)* undef + ret void +} + +; GCN-LABEL: {{^}}v_interp_mov: +; GCN: s_movk_i32 m0, 0x100 +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr0.x{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p20, attr0.x{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, invalid_param_3, attr0.x{{$}} + +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr0.x{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr0.z{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr0.w{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr0.x{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, invalid_param_8, attr0.x{{$}} + +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr63.y{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, p10, attr64.y{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, invalid_param_3, attr64.y{{$}} +; GCN-DAG: v_interp_mov_f32 v{{[0-9]+}}, invalid_param_10, attr64.x{{$}} +define amdgpu_ps void @v_interp_mov(float %x, float %j) { + %mov_0 = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 256) + %mov_1 = call float @llvm.amdgcn.interp.mov(i32 1, i32 0, i32 0, i32 256) + %mov_2 = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 256) + %mov_3 = call float @llvm.amdgcn.interp.mov(i32 3, i32 0, i32 0, i32 256) + + %mov_4 = call float @llvm.amdgcn.interp.mov(i32 0, i32 1, i32 0, i32 256) + %mov_5 = call float @llvm.amdgcn.interp.mov(i32 0, i32 2, i32 0, i32 256) + %mov_6 = call float @llvm.amdgcn.interp.mov(i32 0, i32 3, i32 0, i32 256) + %mov_7 = call float @llvm.amdgcn.interp.mov(i32 0, i32 4, i32 0, i32 256) + %mov_8 = call float @llvm.amdgcn.interp.mov(i32 8, i32 4, i32 0, i32 256) + + %mov_9 = call float @llvm.amdgcn.interp.mov(i32 0, i32 1, i32 63, i32 256) + %mov_10 = call float @llvm.amdgcn.interp.mov(i32 0, i32 1, i32 64, i32 256) + %mov_11 = call float @llvm.amdgcn.interp.mov(i32 3, i32 1, i32 64, i32 256) + %mov_12 = call float @llvm.amdgcn.interp.mov(i32 10, i32 4, i32 64, i32 256) + + store volatile float %mov_0, float addrspace(1)* undef + store volatile float %mov_1, float addrspace(1)* undef + store volatile float %mov_2, float addrspace(1)* undef + store volatile float %mov_3, float addrspace(1)* undef + + store volatile float %mov_4, float addrspace(1)* undef + store volatile float %mov_5, float addrspace(1)* undef + store volatile float %mov_6, float addrspace(1)* undef + store volatile float %mov_7, float addrspace(1)* undef + store volatile float %mov_8, float addrspace(1)* undef + + store volatile float %mov_9, float addrspace(1)* undef + store volatile float %mov_10, float addrspace(1)* undef + store volatile float %mov_11, float addrspace(1)* undef + store volatile float %mov_12, float addrspace(1)* undef + ret void +} + +; SI won't merge ds memory operations, because of the signed offset bug, so +; we only have check lines for VI. +; VI-LABEL: v_interp_readnone: +; VI: s_mov_b32 m0, 0 +; VI-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0 +; VI-DAG: v_interp_mov_f32 v{{[0-9]+}}, p0, attr0.x{{$}} +; VI: s_mov_b32 m0, -1{{$}} +; VI: ds_write2_b32 v{{[0-9]+}}, [[ZERO]], [[ZERO]] offset1:4 +define amdgpu_ps void @v_interp_readnone(float addrspace(3)* %lds) { + store float 0.0, float addrspace(3)* %lds + %tmp1 = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 0) + %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4 + store float 0.0, float addrspace(3)* %tmp2 + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %tmp1, float %tmp1, float %tmp1, float %tmp1) ret void } ; Function Attrs: nounwind readnone -declare float @llvm.amdgcn.interp.p1(i32, i32, i32, i32) #0 +declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #0 ; Function Attrs: nounwind readnone -declare float @llvm.amdgcn.interp.p2(float, i32, i32, i32, i32) #0 +declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #0 + +declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #0 declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll index 07650d990f3c..5d4d4cd7ee46 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.kernarg.segment.ptr.ll @@ -1,11 +1,13 @@ -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=HSA -check-prefix=ALL %s -; RUN: llc -mtriple=amdgcn-mesa-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefix=MESA -check-prefix=ALL %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL,HSA-NOENV %s +; RUN: llc -mtriple=amdgcn--amdhsa-opencl -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,HSA,ALL,HSA-OPENCL %s +; RUN: llc -mtriple=amdgcn-mesa-mesa3d -verify-machineinstrs < %s | FileCheck -check-prefixes=CO-V2,OS-MESA3D,MESA,ALL %s +; RUN: llc -mtriple=amdgcn-mesa-unknown -verify-machineinstrs < %s | FileCheck -check-prefixes=OS-UNKNOWN,MESA,ALL %s ; ALL-LABEL: {{^}}test: -; HSA: enable_sgpr_kernarg_segment_ptr = 1 -; HSA: s_load_dword s{{[0-9]+}}, s[4:5], 0xa +; CO-V2: enable_sgpr_kernarg_segment_ptr = 1 +; CO-V2: s_load_dword s{{[0-9]+}}, s[4:5], 0xa -; MESA: s_load_dword s{{[0-9]+}}, s[0:1], 0xa +; OS-UNKNOWN: s_load_dword s{{[0-9]+}}, s[0:1], 0xa define void @test(i32 addrspace(1)* %out) #1 { %kernarg.segment.ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() %header.ptr = bitcast i8 addrspace(2)* %kernarg.segment.ptr to i32 addrspace(2)* @@ -17,7 +19,7 @@ define void @test(i32 addrspace(1)* %out) #1 { ; ALL-LABEL: {{^}}test_implicit: ; 10 + 9 (36 prepended implicit bytes) + 2(out pointer) = 21 = 0x15 -; MESA: s_load_dword s{{[0-9]+}}, s[0:1], 0x15 +; OS-UNKNOWN: s_load_dword s{{[0-9]+}}, s[0:1], 0x15 define void @test_implicit(i32 addrspace(1)* %out) #1 { %implicitarg.ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr() %header.ptr = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)* @@ -27,6 +29,24 @@ define void @test_implicit(i32 addrspace(1)* %out) #1 { ret void } +; ALL-LABEL: {{^}}test_implicit_alignment +; HSA-NOENV: kernarg_segment_byte_size = 10 +; HSA-OPENCL: kernarg_segment_byte_size = 48 +; OS-MESA3D: kernarg_segment_byte_size = 28 +; OS-UNKNOWN: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xc +; HSA: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x4 +; OS-MESA3D: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x3 +; ALL: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[VAL]] +; MESA: buffer_store_dword [[V_VAL]] +; HSA: flat_store_dword v[{{[0-9]+:[0-9]+}}], [[V_VAL]] +define void @test_implicit_alignment(i32 addrspace(1)* %out, <2 x i8> %in) #1 { + %implicitarg.ptr = call noalias i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr() + %arg.ptr = bitcast i8 addrspace(2)* %implicitarg.ptr to i32 addrspace(2)* + %val = load i32, i32 addrspace(2)* %arg.ptr + store i32 %val, i32 addrspace(1)* %out + ret void +} + declare i8 addrspace(2)* @llvm.amdgcn.kernarg.segment.ptr() #0 declare i8 addrspace(2)* @llvm.amdgcn.implicitarg.ptr() #0 diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ldexp.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ldexp.f16.ll new file mode 100644 index 000000000000..a4b8d7fa58da --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ldexp.f16.ll @@ -0,0 +1,45 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.ldexp.f16(half %a, i32 %b) + +; GCN-LABEL: {{^}}ldexp_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_I32:[0-9]+]] +; VI: v_ldexp_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_I32]] +; GCN: buffer_store_short v[[R_F16]] +define void @ldexp_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + i32 addrspace(1)* %b) { + %a.val = load half, half addrspace(1)* %a + %b.val = load i32, i32 addrspace(1)* %b + %r.val = call half @llvm.amdgcn.ldexp.f16(half %a.val, i32 %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}ldexp_f16_imm_a +; GCN: buffer_load_dword v[[B_I32:[0-9]+]] +; VI: v_ldexp_f16_e32 v[[R_F16:[0-9]+]], 2.0, v[[B_I32]] +; GCN: buffer_store_short v[[R_F16]] +define void @ldexp_f16_imm_a( + half addrspace(1)* %r, + i32 addrspace(1)* %b) { + %b.val = load i32, i32 addrspace(1)* %b + %r.val = call half @llvm.amdgcn.ldexp.f16(half 2.0, i32 %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}ldexp_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_ldexp_f16_e64 v[[R_F16:[0-9]+]], v[[A_F16]], 2{{$}} +; GCN: buffer_store_short v[[R_F16]] +define void @ldexp_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.ldexp.f16(half %a.val, i32 2) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll new file mode 100644 index 000000000000..7c2495e096ec --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.pk.u16.u8.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64, i32, i64) #0 + +; GCN-LABEL: {{^}}v_mqsad_pk_u16_u8: +; GCN: v_mqsad_pk_u16_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) { + %result= call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src, i32 100, i64 100) #0 + store i64 %result, i64 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_mqsad_pk_u16_u8_non_immediate: +; GCN: v_mqsad_pk_u16_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_pk_u16_u8_non_immediate(i64 addrspace(1)* %out, i64 %src, i32 %a, i64 %b) { + %result= call i64 @llvm.amdgcn.mqsad.pk.u16.u8(i64 %src, i32 %a, i64 %b) #0 + store i64 %result, i64 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.u32.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.u32.u8.ll new file mode 100644 index 000000000000..04bb97a9eb57 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.mqsad.u32.u8.ll @@ -0,0 +1,48 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64, i32, <4 x i32>) #0 + +; GCN-LABEL: {{^}}v_mqsad_u32_u8_use_non_inline_constant: +; GCN: v_mqsad_u32_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_u32_u8_use_non_inline_constant(<4 x i32> addrspace(1)* %out, i64 %src) { + %result = call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src, i32 100, <4 x i32> <i32 100, i32 100, i32 100, i32 100>) #0 + store <4 x i32> %result, <4 x i32> addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_mqsad_u32_u8_non_immediate: +; GCN: v_mqsad_u32_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_u32_u8_non_immediate(<4 x i32> addrspace(1)* %out, i64 %src, i32 %a, <4 x i32> %b) { + %result = call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src, i32 %a, <4 x i32> %b) #0 + store <4 x i32> %result, <4 x i32> addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_mqsad_u32_u8_inline_integer_immediate: +; GCN: v_mqsad_u32_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_u32_u8_inline_integer_immediate(<4 x i32> addrspace(1)* %out, i64 %src, i32 %a) { + %result = call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src, i32 %a, <4 x i32> <i32 10, i32 20, i32 30, i32 40>) #0 + store <4 x i32> %result, <4 x i32> addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_mqsad_u32_u8_inline_fp_immediate: +; GCN: v_mqsad_u32_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_u32_u8_inline_fp_immediate(<4 x i32> addrspace(1)* %out, i64 %src, i32 %a) { + %result = call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src, i32 %a, <4 x i32> <i32 1065353216, i32 0, i32 0, i32 0>) #0 + store <4 x i32> %result, <4 x i32> addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_mqsad_u32_u8_use_sgpr_vgpr: +; GCN: v_mqsad_u32_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_mqsad_u32_u8_use_sgpr_vgpr(<4 x i32> addrspace(1)* %out, i64 %src, i32 %a, <4 x i32> addrspace(1)* %input) { + %in = load <4 x i32>, <4 x i32> addrspace(1) * %input + + %result = call <4 x i32> @llvm.amdgcn.mqsad.u32.u8(i64 %src, i32 %a, <4 x i32> %in) #0 + store <4 x i32> %result, <4 x i32> addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.msad.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.msad.u8.ll new file mode 100644 index 000000000000..83d13ab26846 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.msad.u8.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.msad.u8(i32, i32, i32) #0 + +; GCN-LABEL: {{^}}v_msad_u8: +; GCN: v_msad_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_msad_u8(i32 addrspace(1)* %out, i32 %src) { + %result= call i32 @llvm.amdgcn.msad.u8(i32 %src, i32 100, i32 100) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_msad_u8_non_immediate: +; GCN: v_msad_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_msad_u8_non_immediate(i32 addrspace(1)* %out, i32 %src, i32 %a, i32 %b) { + %result= call i32 @llvm.amdgcn.msad.u8(i32 %src, i32 %a, i32 %b) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll new file mode 100644 index 000000000000..ece4224f6e67 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.qsad.pk.u16.u8.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i64 @llvm.amdgcn.qsad.pk.u16.u8(i64, i32, i64) #0 + +; GCN-LABEL: {{^}}v_qsad_pk_u16_u8: +; GCN: v_qsad_pk_u16_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_qsad_pk_u16_u8(i64 addrspace(1)* %out, i64 %src) { + %result= call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src, i32 100, i64 100) #0 + store i64 %result, i64 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_qsad_pk_u16_u8_non_immediate: +; GCN: v_qsad_pk_u16_u8 v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}] +define void @v_qsad_pk_u16_u8_non_immediate(i64 addrspace(1)* %out, i64 %src, i32 %a, i64 %b) { + %result= call i64 @llvm.amdgcn.qsad.pk.u16.u8(i64 %src, i32 %a, i64 %b) #0 + store i64 %result, i64 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll new file mode 100644 index 000000000000..3fda4a59df29 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.rcp.f16(half %a) + +; GCN-LABEL: {{^}}rcp_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_rcp_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @rcp_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.rcp.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.legacy.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.legacy.ll new file mode 100644 index 000000000000..d53861456c78 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.legacy.ll @@ -0,0 +1,42 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: not llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR %s + +; ERROR: error: <unknown>:0:0: in function rcp_legacy_f32 void (float addrspace(1)*, float): intrinsic not supported on subtarget + +declare float @llvm.amdgcn.rcp.legacy(float) #0 + +; GCN-LABEL: {{^}}rcp_legacy_f32: +; GCN: v_rcp_legacy_f32_e32 {{v[0-9]+}}, {{s[0-9]+}} +define void @rcp_legacy_f32(float addrspace(1)* %out, float %src) #1 { + %rcp = call float @llvm.amdgcn.rcp.legacy(float %src) #0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; TODO: Really these should be constant folded +; GCN-LABEL: {{^}}rcp_legacy_f32_constant_4.0 +; GCN: v_rcp_legacy_f32_e32 {{v[0-9]+}}, 4.0 +define void @rcp_legacy_f32_constant_4.0(float addrspace(1)* %out) #1 { + %rcp = call float @llvm.amdgcn.rcp.legacy(float 4.0) #0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}rcp_legacy_f32_constant_100.0 +; GCN: v_rcp_legacy_f32_e32 {{v[0-9]+}}, 0x42c80000 +define void @rcp_legacy_f32_constant_100.0(float addrspace(1)* %out) #1 { + %rcp = call float @llvm.amdgcn.rcp.legacy(float 100.0) #0 + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}rcp_legacy_undef_f32: +; GCN-NOT: v_rcp_legacy_f32 +define void @rcp_legacy_undef_f32(float addrspace(1)* %out) #1 { + %rcp = call float @llvm.amdgcn.rcp.legacy(float undef) + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.read.workdim.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.read.workdim.ll deleted file mode 100644 index 76a5757e4c20..000000000000 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.read.workdim.ll +++ /dev/null @@ -1,46 +0,0 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=SI-NOHSA -check-prefix=GCN-NOHSA %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=VI-NOHSA -check-prefix=GCN -check-prefix=GCN-NOHSA %s - -; GCN-LABEL: {{^}}read_workdim: -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0xb -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x2c -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NOHSA: buffer_store_dword [[VVAL]] -define void @read_workdim(i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.amdgcn.read.workdim() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -; GCN-LABEL: {{^}}read_workdim_known_bits: -; SI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0xb -; VI: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x2c -; GCN-NOT: 0xff -; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN: buffer_store_dword [[VVAL]] -define void @read_workdim_known_bits(i32 addrspace(1)* %out) { -entry: - %dim = call i32 @llvm.amdgcn.read.workdim() #0 - %shl = shl i32 %dim, 24 - %shr = lshr i32 %shl, 24 - store i32 %shr, i32 addrspace(1)* %out - ret void -} - -; GCN-LABEL: {{^}}legacy_read_workdim: -; SI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0xb -; VI-NOHSA: s_load_dword [[VAL:s[0-9]+]], s[0:1], 0x2c -; GCN-NOHSA: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[VAL]] -; GCN-NOHSA: buffer_store_dword [[VVAL]] -define void @legacy_read_workdim(i32 addrspace(1)* %out) { -entry: - %dim = call i32 @llvm.AMDGPU.read.workdim() #0 - store i32 %dim, i32 addrspace(1)* %out - ret void -} - -declare i32 @llvm.amdgcn.read.workdim() #0 -declare i32 @llvm.AMDGPU.read.workdim() #0 - -attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll new file mode 100644 index 000000000000..2569108e7b18 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll @@ -0,0 +1,36 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s + +declare i32 @llvm.amdgcn.readfirstlane(i32) #0 + +; CHECK-LABEL: {{^}}test_readfirstlane: +; CHECK: v_readfirstlane_b32 s{{[0-9]+}}, v{{[0-9]+}} +define void @test_readfirstlane(i32 addrspace(1)* %out, i32 %src) #1 { + %readfirstlane = call i32 @llvm.amdgcn.readfirstlane(i32 %src) + store i32 %readfirstlane, i32 addrspace(1)* %out, align 4 + ret void +} + +; CHECK-LABEL: {{^}}test_readfirstlane_imm: +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], 32 +; CHECK: v_readfirstlane_b32 s{{[0-9]+}}, [[VVAL]] +define void @test_readfirstlane_imm(i32 addrspace(1)* %out) #1 { + %readfirstlane = call i32 @llvm.amdgcn.readfirstlane(i32 32) + store i32 %readfirstlane, i32 addrspace(1)* %out, align 4 + ret void +} + +; TODO: m0 should be folded. +; CHECK-LABEL: {{^}}test_readfirstlane_m0: +; CHECK: s_mov_b32 m0, -1 +; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0 +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], [[COPY_M0]] +; CHECK: v_readfirstlane_b32 s{{[0-9]+}}, [[VVAL]] +define void @test_readfirstlane_m0(i32 addrspace(1)* %out) #1 { + %m0 = call i32 asm "s_mov_b32 m0, -1", "={M0}"() + %readfirstlane = call i32 @llvm.amdgcn.readfirstlane(i32 %m0) + store i32 %readfirstlane, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone convergent } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll new file mode 100644 index 000000000000..a9d52b006df0 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.readlane.ll @@ -0,0 +1,44 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s + +declare i32 @llvm.amdgcn.readlane(i32, i32) #0 + +; CHECK-LABEL: {{^}}test_readlane_sreg: +; CHECK: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +define void @test_readlane_sreg(i32 addrspace(1)* %out, i32 %src0, i32 %src1) #1 { + %readlane = call i32 @llvm.amdgcn.readlane(i32 %src0, i32 %src1) + store i32 %readlane, i32 addrspace(1)* %out, align 4 + ret void +} + +; CHECK-LABEL: {{^}}test_readlane_imm_sreg: +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], 32 +; CHECK: v_readlane_b32 s{{[0-9]+}}, [[VVAL]], s{{[0-9]+}} +define void @test_readlane_imm_sreg(i32 addrspace(1)* %out, i32 %src1) #1 { + %readlane = call i32 @llvm.amdgcn.readlane(i32 32, i32 %src1) + store i32 %readlane, i32 addrspace(1)* %out, align 4 + ret void +} + +; TODO: m0 should be folded. +; CHECK-LABEL: {{^}}test_readlane_m0_sreg: +; CHECK: s_mov_b32 m0, -1 +; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0 +; CHECK: v_mov_b32_e32 [[VVAL:v[0-9]]], [[COPY_M0]] +; CHECK: v_readlane_b32 s{{[0-9]+}}, [[VVAL]], s{{[0-9]+}} +define void @test_readlane_m0_sreg(i32 addrspace(1)* %out, i32 %src1) #1 { + %m0 = call i32 asm "s_mov_b32 m0, -1", "={M0}"() + %readlane = call i32 @llvm.amdgcn.readlane(i32 %m0, i32 %src1) + store i32 %readlane, i32 addrspace(1)* %out, align 4 + ret void +} + +; CHECK-LABEL: {{^}}test_readlane_imm: +; CHECK: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 32 +define void @test_readlane_imm(i32 addrspace(1)* %out, i32 %src0) #1 { + %readlane = call i32 @llvm.amdgcn.readlane(i32 %src0, i32 32) #0 + store i32 %readlane, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone convergent } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll index 73a5c54e175e..7b1373b13f3a 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll @@ -10,9 +10,7 @@ declare double @llvm.amdgcn.rsq.clamp.f64(double) #1 ; VI: s_load_dword [[SRC:s[0-9]+]] ; VI-DAG: v_rsq_f32_e32 [[RSQ:v[0-9]+]], [[SRC]] ; VI-DAG: v_min_f32_e32 [[MIN:v[0-9]+]], 0x7f7fffff, [[RSQ]] -; TODO: this constant should be folded: -; VI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 0xff7fffff -; VI: v_max_f32_e32 [[RESULT:v[0-9]+]], [[MIN]], [[K]] +; VI: v_max_f32_e32 [[RESULT:v[0-9]+]], 0xff7fffff, [[MIN]] ; VI: buffer_store_dword [[RESULT]] define void @rsq_clamp_f32(float addrspace(1)* %out, float %src) #0 { %rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src) @@ -25,7 +23,8 @@ define void @rsq_clamp_f32(float addrspace(1)* %out, float %src) #0 { ; SI: v_rsq_clamp_f64_e32 ; TODO: this constant should be folded: -; VI-DAG: s_mov_b32 s[[LOW1:[0-9+]]], -1 +; VI-DAG: s_mov_b32 [[NEG1:s[0-9+]]], -1 +; VI-DAG: s_mov_b32 s[[LOW1:[0-9+]]], [[NEG1]] ; VI-DAG: s_mov_b32 s[[HIGH1:[0-9+]]], 0x7fefffff ; VI-DAG: s_mov_b32 s[[HIGH2:[0-9+]]], 0xffefffff ; VI-DAG: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], s[{{[0-9]+:[0-9]+}} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll new file mode 100644 index 000000000000..74415e3658a4 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.rsq.f16(half %a) + +; GCN-LABEL: {{^}}rsq_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_rsq_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @rsq_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.rsq.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.decperflevel.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.decperflevel.ll new file mode 100644 index 000000000000..72513fc86f49 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.decperflevel.ll @@ -0,0 +1,43 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare void @llvm.amdgcn.s.decperflevel(i32) #0 + +; GCN-LABEL: {{^}}test_s_decperflevel: +; GCN: s_decperflevel 0{{$}} +; GCN: s_decperflevel 1{{$}} +; GCN: s_decperflevel 2{{$}} +; GCN: s_decperflevel 3{{$}} +; GCN: s_decperflevel 4{{$}} +; GCN: s_decperflevel 5{{$}} +; GCN: s_decperflevel 6{{$}} +; GCN: s_decperflevel 7{{$}} +; GCN: s_decperflevel 8{{$}} +; GCN: s_decperflevel 9{{$}} +; GCN: s_decperflevel 10{{$}} +; GCN: s_decperflevel 11{{$}} +; GCN: s_decperflevel 12{{$}} +; GCN: s_decperflevel 13{{$}} +; GCN: s_decperflevel 14{{$}} +; GCN: s_decperflevel 15{{$}} +define void @test_s_decperflevel(i32 %x) #0 { + call void @llvm.amdgcn.s.decperflevel(i32 0) + call void @llvm.amdgcn.s.decperflevel(i32 1) + call void @llvm.amdgcn.s.decperflevel(i32 2) + call void @llvm.amdgcn.s.decperflevel(i32 3) + call void @llvm.amdgcn.s.decperflevel(i32 4) + call void @llvm.amdgcn.s.decperflevel(i32 5) + call void @llvm.amdgcn.s.decperflevel(i32 6) + call void @llvm.amdgcn.s.decperflevel(i32 7) + call void @llvm.amdgcn.s.decperflevel(i32 8) + call void @llvm.amdgcn.s.decperflevel(i32 9) + call void @llvm.amdgcn.s.decperflevel(i32 10) + call void @llvm.amdgcn.s.decperflevel(i32 11) + call void @llvm.amdgcn.s.decperflevel(i32 12) + call void @llvm.amdgcn.s.decperflevel(i32 13) + call void @llvm.amdgcn.s.decperflevel(i32 14) + call void @llvm.amdgcn.s.decperflevel(i32 15) + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll index 251eec656edc..4304398182a6 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.getreg.ll @@ -1,11 +1,21 @@ -; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s -; FUNC-LABEL: {{^}}s_getreg_test: -; CHECK: s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_LDS_ALLOC, 8, 23) +; GCN-LABEL: {{^}}s_getreg_test: +; GCN: s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_LDS_ALLOC, 8, 23) define void @s_getreg_test(i32 addrspace(1)* %out) { ; simm16=45574 for lds size. - %lds_size_64dwords = call i32 @llvm.amdgcn.s.getreg(i32 45574) #0 + %lds_size_64dwords = call i32 @llvm.amdgcn.s.getreg(i32 45574) + %lds_size_bytes = shl i32 %lds_size_64dwords, 8 + store i32 %lds_size_bytes, i32 addrspace(1)* %out + ret void +} + +; Call site has additional readnone knowledge. +; GCN-LABEL: {{^}}readnone_s_getreg_test: +; GCN: s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_LDS_ALLOC, 8, 23) +define void @readnone_s_getreg_test(i32 addrspace(1)* %out) { ; simm16=45574 for lds size. + %lds_size_64dwords = call i32 @llvm.amdgcn.s.getreg(i32 45574) #1 %lds_size_bytes = shl i32 %lds_size_64dwords, 8 store i32 %lds_size_bytes, i32 addrspace(1)* %out ret void @@ -13,4 +23,5 @@ define void @s_getreg_test(i32 addrspace(1)* %out) { ; simm16=45574 for lds size declare i32 @llvm.amdgcn.s.getreg(i32) #0 -attributes #0 = { nounwind readonly} +attributes #0 = { nounwind readonly } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.incperflevel.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.incperflevel.ll new file mode 100644 index 000000000000..2ae4fc473eaa --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.incperflevel.ll @@ -0,0 +1,43 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare void @llvm.amdgcn.s.incperflevel(i32) #0 + +; GCN-LABEL: {{^}}test_s_incperflevel: +; GCN: s_incperflevel 0{{$}} +; GCN: s_incperflevel 1{{$}} +; GCN: s_incperflevel 2{{$}} +; GCN: s_incperflevel 3{{$}} +; GCN: s_incperflevel 4{{$}} +; GCN: s_incperflevel 5{{$}} +; GCN: s_incperflevel 6{{$}} +; GCN: s_incperflevel 7{{$}} +; GCN: s_incperflevel 8{{$}} +; GCN: s_incperflevel 9{{$}} +; GCN: s_incperflevel 10{{$}} +; GCN: s_incperflevel 11{{$}} +; GCN: s_incperflevel 12{{$}} +; GCN: s_incperflevel 13{{$}} +; GCN: s_incperflevel 14{{$}} +; GCN: s_incperflevel 15{{$}} +define void @test_s_incperflevel(i32 %x) #0 { + call void @llvm.amdgcn.s.incperflevel(i32 0) + call void @llvm.amdgcn.s.incperflevel(i32 1) + call void @llvm.amdgcn.s.incperflevel(i32 2) + call void @llvm.amdgcn.s.incperflevel(i32 3) + call void @llvm.amdgcn.s.incperflevel(i32 4) + call void @llvm.amdgcn.s.incperflevel(i32 5) + call void @llvm.amdgcn.s.incperflevel(i32 6) + call void @llvm.amdgcn.s.incperflevel(i32 7) + call void @llvm.amdgcn.s.incperflevel(i32 8) + call void @llvm.amdgcn.s.incperflevel(i32 9) + call void @llvm.amdgcn.s.incperflevel(i32 10) + call void @llvm.amdgcn.s.incperflevel(i32 11) + call void @llvm.amdgcn.s.incperflevel(i32 12) + call void @llvm.amdgcn.s.incperflevel(i32 13) + call void @llvm.amdgcn.s.incperflevel(i32 14) + call void @llvm.amdgcn.s.incperflevel(i32 15) + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll index c2d48f99aac5..d73986ac5753 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.s.waitcnt.ll @@ -7,9 +7,9 @@ ; CHECK-NEXT: image_store ; CHECK-NEXT: s_endpgm define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <4 x float> %d0, <4 x float> %d1, i32 %c0, i32 %c1) { - call void @llvm.amdgcn.image.store.i32(<4 x float> %d0, i32 %c0, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 1, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %d0, i32 %c0, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 1, i1 0) call void @llvm.amdgcn.s.waitcnt(i32 3840) ; 0xf00 - call void @llvm.amdgcn.image.store.i32(<4 x float> %d1, i32 %c1, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 1, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %d1, i32 %c1, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 1, i1 0) ret void } @@ -22,17 +22,17 @@ define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <4 x float> %d0, <4 x float> ; CHECK: s_waitcnt ; CHECK-NEXT: image_store define amdgpu_ps void @test2(<8 x i32> inreg %rsrc, i32 %c) { - %t = call <4 x float> @llvm.amdgcn.image.load.i32(i32 %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %t = call <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32 %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) call void @llvm.amdgcn.s.waitcnt(i32 3840) ; 0xf00 %c.1 = mul i32 %c, 2 - call void @llvm.amdgcn.image.store.i32(<4 x float> %t, i32 %c.1, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float> %t, i32 %c.1, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret void } declare void @llvm.amdgcn.s.waitcnt(i32) #0 -declare <4 x float> @llvm.amdgcn.image.load.i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #1 -declare void @llvm.amdgcn.image.store.i32(<4 x float>, i32, <8 x i32>, i32, i1, i1, i1, i1) #0 +declare <4 x float> @llvm.amdgcn.image.load.v4f32.i32.v8i32(i32, <8 x i32>, i32, i1, i1, i1, i1) #1 +declare void @llvm.amdgcn.image.store.v4f32.i32.v8i32(<4 x float>, i32, <8 x i32>, i32, i1, i1, i1, i1) #0 attributes #0 = { nounwind } attributes #1 = { nounwind readonly } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sad.hi.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.hi.u8.ll new file mode 100644 index 000000000000..3aaed9d53772 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.hi.u8.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.sad.hi.u8(i32, i32, i32) #0 + +; GCN-LABEL: {{^}}v_sad_hi_u8: +; GCN: v_sad_hi_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_hi_u8(i32 addrspace(1)* %out, i32 %src) { + %result= call i32 @llvm.amdgcn.sad.hi.u8(i32 %src, i32 100, i32 100) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_sad_hi_u8_non_immediate: +; GCN: v_sad_hi_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_hi_u8_non_immediate(i32 addrspace(1)* %out, i32 %src, i32 %a, i32 %b) { + %result= call i32 @llvm.amdgcn.sad.hi.u8(i32 %src, i32 %a, i32 %b) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u16.ll new file mode 100644 index 000000000000..5438571c5821 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u16.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.sad.u16(i32, i32, i32) #0 + +; GCN-LABEL: {{^}}v_sad_u16: +; GCN: v_sad_u16 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u16(i32 addrspace(1)* %out, i32 %src) { + %result= call i32 @llvm.amdgcn.sad.u16(i32 %src, i32 100, i32 100) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_sad_u16_non_immediate: +; GCN: v_sad_u16 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u16_non_immediate(i32 addrspace(1)* %out, i32 %src, i32 %a, i32 %b) { + %result= call i32 @llvm.amdgcn.sad.u16(i32 %src, i32 %a, i32 %b) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u8.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u8.ll new file mode 100644 index 000000000000..9422d7620ca6 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sad.u8.ll @@ -0,0 +1,22 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.sad.u8(i32, i32, i32) #0 + +; GCN-LABEL: {{^}}v_sad_u8: +; GCN: v_sad_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u8(i32 addrspace(1)* %out, i32 %src) { + %result= call i32 @llvm.amdgcn.sad.u8(i32 %src, i32 100, i32 100) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_sad_u8_non_immediate: +; GCN: v_sad_u8 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u8_non_immediate(i32 addrspace(1)* %out, i32 %src, i32 %a, i32 %b) { + %result= call i32 @llvm.amdgcn.sad.u8(i32 %src, i32 %a, i32 %b) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sffbh.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sffbh.ll new file mode 100644 index 000000000000..a348af259435 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sffbh.ll @@ -0,0 +1,54 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.sffbh.i32(i32) #1 +declare i32 @llvm.AMDGPU.flbit.i32(i32) #1 + +; FUNC-LABEL: {{^}}s_flbit: +; GCN: s_load_dword [[VAL:s[0-9]+]], +; GCN: s_flbit_i32 [[SRESULT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] +; GCN: buffer_store_dword [[VRESULT]], +define void @s_flbit(i32 addrspace(1)* noalias %out, i32 %val) #0 { + %r = call i32 @llvm.amdgcn.sffbh.i32(i32 %val) + store i32 %r, i32 addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}v_flbit: +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_i32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], +define void @v_flbit(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) #0 { + %val = load i32, i32 addrspace(1)* %valptr, align 4 + %r = call i32 @llvm.amdgcn.sffbh.i32(i32 %val) + store i32 %r, i32 addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}legacy_s_flbit: +; GCN: s_load_dword [[VAL:s[0-9]+]], +; GCN: s_flbit_i32 [[SRESULT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]] +; GCN: buffer_store_dword [[VRESULT]], +; GCN: s_endpgm +define void @legacy_s_flbit(i32 addrspace(1)* noalias %out, i32 %val) nounwind { + %r = call i32 @llvm.AMDGPU.flbit.i32(i32 %val) nounwind readnone + store i32 %r, i32 addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}legacy_v_flbit: +; GCN: buffer_load_dword [[VAL:v[0-9]+]], +; GCN: v_ffbh_i32_e32 [[RESULT:v[0-9]+]], [[VAL]] +; GCN: buffer_store_dword [[RESULT]], +; GCN: s_endpgm +define void @legacy_v_flbit(i32 addrspace(1)* noalias %out, i32 addrspace(1)* noalias %valptr) nounwind { + %val = load i32, i32 addrspace(1)* %valptr, align 4 + %r = call i32 @llvm.AMDGPU.flbit.i32(i32 %val) nounwind readnone + store i32 %r, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.sin.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.sin.f16.ll new file mode 100644 index 000000000000..0ebe012fe1c5 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.sin.f16.ll @@ -0,0 +1,18 @@ +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.amdgcn.sin.f16(half %a) + +; GCN-LABEL: {{^}}sin_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; VI: v_sin_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @sin_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.amdgcn.sin.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.wave.barrier.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.wave.barrier.ll new file mode 100644 index 000000000000..e85179755371 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.wave.barrier.ll @@ -0,0 +1,16 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}test_wave_barrier: +; GCN-DAG: ; wave barrier +; GCN-NOT: s_barrier + +define void @test_wave_barrier() #0 { +entry: + call void @llvm.amdgcn.wave.barrier() #1 + ret void +} + +declare void @llvm.amdgcn.wave.barrier() #1 + +attributes #0 = { nounwind } +attributes #1 = { convergent nounwind } diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll index c22eac7e271c..58529b874442 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.workgroup.id.ll @@ -1,7 +1,9 @@ -; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=HSA -check-prefix=CI-HSA %s -; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=HSA -check-prefix=VI-HSA %s -; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=SI-MESA %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=VI-MESA %s +; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=CI-HSA %s +; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=VI-HSA %s +; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=UNKNOWN-OS -check-prefix=SI-MESA %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=UNKNOWN-OS -check-prefix=VI-MESA %s +; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s +; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,VI-MESA %s declare i32 @llvm.amdgcn.workgroup.id.x() #0 declare i32 @llvm.amdgcn.workgroup.id.y() #0 @@ -9,26 +11,25 @@ declare i32 @llvm.amdgcn.workgroup.id.z() #0 ; ALL-LABEL {{^}}test_workgroup_id_x: -; HSA: .amd_kernel_code_t -; HSA: compute_pgm_rsrc2_user_sgpr = 6 -; HSA: compute_pgm_rsrc2_tgid_x_en = 1 -; HSA: compute_pgm_rsrc2_tgid_y_en = 0 -; HSA: compute_pgm_rsrc2_tgid_z_en = 0 -; HSA: compute_pgm_rsrc2_tg_size_en = 0 -; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 0 -; HSA: enable_sgpr_grid_workgroup_count_x = 0 -; HSA: enable_sgpr_grid_workgroup_count_y = 0 -; HSA: enable_sgpr_grid_workgroup_count_z = 0 -; HSA: .end_amd_kernel_code_t +; CO-V2: .amd_kernel_code_t +; CO-V2: user_sgpr_count = 6 +; CO-V2: enable_sgpr_workgroup_id_x = 1 +; CO-V2: enable_sgpr_workgroup_id_y = 0 +; CO-V2: enable_sgpr_workgroup_id_z = 0 +; CO-V2: enable_sgpr_workgroup_info = 0 +; CO-V2: enable_vgpr_workitem_id = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_x = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_y = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_z = 0 +; CO-V2: .end_amd_kernel_code_t -; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s2{{$}} -; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s6{{$}} +; UNKNOWN-OS: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s2{{$}} +; CO-V2: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s6{{$}} -; ALL-NOT: [[VCOPY]] ; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]] -; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6 -; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 +; CO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 6 +; ALL-NOCO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 2 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 @@ -40,23 +41,22 @@ define void @test_workgroup_id_x(i32 addrspace(1)* %out) #1 { } ; ALL-LABEL {{^}}test_workgroup_id_y: -; HSA: compute_pgm_rsrc2_user_sgpr = 6 -; HSA: compute_pgm_rsrc2_tgid_x_en = 1 -; HSA: compute_pgm_rsrc2_tgid_y_en = 1 -; HSA: compute_pgm_rsrc2_tgid_z_en = 0 -; HSA: compute_pgm_rsrc2_tg_size_en = 0 -; HSA: enable_sgpr_grid_workgroup_count_x = 0 -; HSA: enable_sgpr_grid_workgroup_count_y = 0 -; HSA: enable_sgpr_grid_workgroup_count_z = 0 +; CO-V2: user_sgpr_count = 6 +; CO-V2: enable_sgpr_workgroup_id_x = 1 +; CO-V2: enable_sgpr_workgroup_id_y = 1 +; CO-V2: enable_sgpr_workgroup_id_z = 0 +; CO-V2: enable_sgpr_workgroup_info = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_x = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_y = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_z = 0 -; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}} +; UNKNOWN-OS: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}} ; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s7{{$}} -; ALL-NOT: [[VCOPY]] ; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]] -; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6 -; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 +; CO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 6 +; ALL-NOCO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 2 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 1 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 0 @@ -68,31 +68,30 @@ define void @test_workgroup_id_y(i32 addrspace(1)* %out) #1 { } ; ALL-LABEL {{^}}test_workgroup_id_z: -; HSA: compute_pgm_rsrc2_user_sgpr = 6 -; HSA: compute_pgm_rsrc2_tgid_x_en = 1 -; HSA: compute_pgm_rsrc2_tgid_y_en = 0 -; HSA: compute_pgm_rsrc2_tgid_z_en = 1 -; HSA: compute_pgm_rsrc2_tg_size_en = 0 -; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 0 -; HSA: enable_sgpr_private_segment_buffer = 1 -; HSA: enable_sgpr_dispatch_ptr = 0 -; HSA: enable_sgpr_queue_ptr = 0 -; HSA: enable_sgpr_kernarg_segment_ptr = 1 -; HSA: enable_sgpr_dispatch_id = 0 -; HSA: enable_sgpr_flat_scratch_init = 0 -; HSA: enable_sgpr_private_segment_size = 0 -; HSA: enable_sgpr_grid_workgroup_count_x = 0 -; HSA: enable_sgpr_grid_workgroup_count_y = 0 -; HSA: enable_sgpr_grid_workgroup_count_z = 0 +; CO-V2: user_sgpr_count = 6 +; CO-V2: enable_sgpr_workgroup_id_x = 1 +; CO-V2: enable_sgpr_workgroup_id_y = 0 +; CO-V2: enable_sgpr_workgroup_id_z = 1 +; CO-V2: enable_sgpr_workgroup_info = 0 +; CO-V2: enable_vgpr_workitem_id = 0 +; CO-V2: enable_sgpr_private_segment_buffer = 1 +; CO-V2: enable_sgpr_dispatch_ptr = 0 +; CO-V2: enable_sgpr_queue_ptr = 0 +; CO-V2: enable_sgpr_kernarg_segment_ptr = 1 +; CO-V2: enable_sgpr_dispatch_id = 0 +; CO-V2: enable_sgpr_flat_scratch_init = 0 +; CO-V2: enable_sgpr_private_segment_size = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_x = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_y = 0 +; CO-V2: enable_sgpr_grid_workgroup_count_z = 0 -; MESA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}} +; UNKNOWN-OS: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s3{{$}} ; HSA: v_mov_b32_e32 [[VCOPY:v[0-9]+]], s7{{$}} -; ALL-NOT: [[VCOPY]] ; ALL: {{buffer|flat}}_store_dword {{.*}}[[VCOPY]] -; HSA: COMPUTE_PGM_RSRC2:USER_SGPR: 6 -; ALL-NOHSA: COMPUTE_PGM_RSRC2:USER_SGPR: 2 +; CO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 6 +; ALL-NOCO-V2: COMPUTE_PGM_RSRC2:USER_SGPR: 2 ; ALL: COMPUTE_PGM_RSRC2:TGID_X_EN: 1 ; ALL: COMPUTE_PGM_RSRC2:TGID_Y_EN: 0 ; ALL: COMPUTE_PGM_RSRC2:TGID_Z_EN: 1 diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll index 28ef7b82ef84..8fc12890fad0 100644 --- a/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll +++ b/test/CodeGen/AMDGPU/llvm.amdgcn.workitem.id.ll @@ -1,7 +1,9 @@ -; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=HSA -check-prefix=CI-HSA %s -; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=HSA -check-prefix=VI-HSA %s +; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=CI-HSA %s +; RUN: llc -march=amdgcn -mtriple=amdgcn-unknown-amdhsa -mcpu=carrizo -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=CO-V2 -check-prefix=VI-HSA %s ; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=SI-MESA %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=MESA -check-prefix=VI-MESA %s +; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,SI-MESA %s +; RUN: llc -mtriple=amdgcn-unknown-mesa3d -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=ALL,CO-V2,VI-MESA %s declare i32 @llvm.amdgcn.workitem.id.x() #0 declare i32 @llvm.amdgcn.workitem.id.y() #0 @@ -12,7 +14,7 @@ declare i32 @llvm.amdgcn.workitem.id.z() #0 ; MESA-NEXT: .long 132{{$}} ; ALL-LABEL {{^}}test_workitem_id_x: -; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 0 +; CO-V2: enable_vgpr_workitem_id = 0 ; ALL-NOT: v0 ; ALL: {{buffer|flat}}_store_dword {{.*}}v0 @@ -27,7 +29,7 @@ define void @test_workitem_id_x(i32 addrspace(1)* %out) #1 { ; MESA-NEXT: .long 2180{{$}} ; ALL-LABEL {{^}}test_workitem_id_y: -; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 1 +; CO-V2: enable_vgpr_workitem_id = 1 ; ALL-NOT: v1 ; ALL: {{buffer|flat}}_store_dword {{.*}}v1 @@ -42,7 +44,7 @@ define void @test_workitem_id_y(i32 addrspace(1)* %out) #1 { ; MESA-NEXT: .long 4228{{$}} ; ALL-LABEL {{^}}test_workitem_id_z: -; HSA: compute_pgm_rsrc2_tidig_comp_cnt = 2 +; CO-V2: enable_vgpr_workitem_id = 2 ; ALL-NOT: v2 ; ALL: {{buffer|flat}}_store_dword {{.*}}v2 diff --git a/test/CodeGen/AMDGPU/llvm.ceil.f16.ll b/test/CodeGen/AMDGPU/llvm.ceil.f16.ll new file mode 100644 index 000000000000..dc984e91eec7 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.ceil.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.ceil.f16(half %a) +declare <2 x half> @llvm.ceil.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}ceil_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_ceil_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_ceil_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @ceil_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.ceil.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}ceil_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_ceil_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_ceil_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_ceil_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_ceil_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @ceil_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.ceil.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.cos.f16.ll b/test/CodeGen/AMDGPU/llvm.cos.f16.ll new file mode 100644 index 000000000000..bb3a5a4dea7b --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.cos.f16.ll @@ -0,0 +1,55 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.cos.f16(half %a) +declare <2 x half> @llvm.cos.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}cos_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_mul_f32_e32 v[[M_F32:[0-9]+]], {{0.15915494|0x3e22f983}}, v[[A_F32]] +; GCN: v_fract_f32_e32 v[[F_F32:[0-9]+]], v[[M_F32]] +; GCN: v_cos_f32_e32 v[[R_F32:[0-9]+]], v[[F_F32]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @cos_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.cos.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}cos_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_mov_b32_e32 v[[HALF_PIE:[0-9]+]], 0x3e22f983{{$}} +; GCN: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_mul_f32_e32 v[[M_F32_0:[0-9]+]], v[[HALF_PIE]], v[[A_F32_0]] +; VI: v_mul_f32_e32 v[[M_F32_0:[0-9]+]], 0.15915494, v[[A_F32_0]] +; GCN: v_fract_f32_e32 v[[F_F32_0:[0-9]+]], v[[M_F32_0]] +; SI: v_mul_f32_e32 v[[M_F32_1:[0-9]+]], v[[HALF_PIE]], v[[A_F32_1]] +; VI: v_mul_f32_e32 v[[M_F32_1:[0-9]+]], 0.15915494, v[[A_F32_1]] +; GCN: v_fract_f32_e32 v[[F_F32_1:[0-9]+]], v[[M_F32_1]] +; GCN: v_cos_f32_e32 v[[R_F32_0:[0-9]+]], v[[F_F32_0]] +; GCN: v_cos_f32_e32 v[[R_F32_1:[0-9]+]], v[[F_F32_1]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @cos_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.cos.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.dbg.value.ll b/test/CodeGen/AMDGPU/llvm.dbg.value.ll index 1a37ba311606..8b0854c2c2b5 100644 --- a/test/CodeGen/AMDGPU/llvm.dbg.value.ll +++ b/test/CodeGen/AMDGPU/llvm.dbg.value.ll @@ -2,7 +2,11 @@ ; CHECK-LABEL: {{^}}test_debug_value: ; CHECK: s_load_dwordx2 s[4:5] -; CHECK: DEBUG_VALUE: test_debug_value:globalptr_arg <- %SGPR4_SGPR5 + +; FIXME: Why is the SGPR4_SGPR5 reference being removed from DBG_VALUE? +; CHECK: ; kill: %SGPR4_SGPR5<def> %SGPR4_SGPR5<kill> +; CHECK-NEXT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- undef + ; CHECK: buffer_store_dword ; CHECK: s_endpgm define void @test_debug_value(i32 addrspace(1)* nocapture %globalptr_arg) #0 !dbg !4 { diff --git a/test/CodeGen/AMDGPU/llvm.exp2.f16.ll b/test/CodeGen/AMDGPU/llvm.exp2.f16.ll new file mode 100644 index 000000000000..9165698cb815 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.exp2.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.exp2.f16(half %a) +declare <2 x half> @llvm.exp2.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}exp2_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_exp_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_exp_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @exp2_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.exp2.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}exp2_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_exp_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_exp_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_exp_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_exp_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @exp2_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.exp2.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.floor.f16.ll b/test/CodeGen/AMDGPU/llvm.floor.f16.ll new file mode 100644 index 000000000000..a7bdb1e9428a --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.floor.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.floor.f16(half %a) +declare <2 x half> @llvm.floor.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}floor_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_floor_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_floor_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @floor_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.floor.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}floor_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_floor_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_floor_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_floor_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_floor_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @floor_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.floor.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.fma.f16.ll b/test/CodeGen/AMDGPU/llvm.fma.f16.ll new file mode 100644 index 000000000000..e6f85ef08f48 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.fma.f16.ll @@ -0,0 +1,235 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.fma.f16(half %a, half %b, half %c) +declare <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c) + +; GCN-LABEL: {{^}}fma_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], v[[C_F32:[0-9]]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fma_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) { + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fma.f16(half %a.val, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], v[[C_F32:[0-9]]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x4200{{$}} +; VI: v_fma_f16 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fma_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b, + half addrspace(1)* %c) { + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fma.f16(half 3.0, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], v[[C_F32:[0-9]]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x4200{{$}} +; VI: v_fma_f16 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fma_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %c) { + %a.val = load half, half addrspace(1)* %a + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fma.f16(half %a.val, half 3.0, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_f16_imm_c +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_fma_f32 v[[R_F32:[0-9]+]], v[[A_F32:[0-9]]], v[[B_F32:[0-9]]], v[[C_F32:[0-9]]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_mov_b32_e32 v[[C_F16:[0-9]+]], 0x4200{{$}} +; VI: v_fma_f16 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]], v[[C_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @fma_f16_imm_c( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.fma.f16(half %a.val, half %b.val, half 3.0) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]] +; SI: v_fma_f32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]], v[[C_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_fma_f32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]], v[[C_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_V2_F16]] +; VI: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fma_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) { + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> %c.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; VI: v_mov_b32_e32 v[[A_F16:[0-9]+]], 0x4200{{$}} +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]] +; SI: v_fma_f32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32]], v[[C_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_fma_f32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32]], v[[C_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_F16]], v[[C_V2_F16]] +; VI: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16]], v[[C_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fma_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) { + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> <half 3.0, half 3.0>, <2 x half> %b.val, <2 x half> %c.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4200{{$}} +; VI: v_mov_b32_e32 v[[B_F16:[0-9]+]], 0x4200{{$}} +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]] +; SI: v_fma_f32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32]], v[[C_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_fma_f32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32]], v[[C_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]], v[[B_F16]], v[[C_V2_F16]] +; VI: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16]], v[[C_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fma_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %c) { + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> <half 3.0, half 3.0>, <2 x half> %c.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fma_v2f16_imm_c +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], 0x4200{{$}} +; VI: v_mov_b32_e32 v[[C_F16:[0-9]+]], 0x4200{{$}} +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_fma_f32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_fma_f32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_fma_f16 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_F16]] +; VI: v_fma_f16 v[[R_F16_1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fma_v2f16_imm_c( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> <half 3.0, half 3.0>) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll b/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll new file mode 100644 index 000000000000..af7bd2703948 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.fmuladd.f16.ll @@ -0,0 +1,145 @@ +; RUN: llc -march=amdgcn -mattr=-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SI-FLUSH %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=VI-FLUSH %s +; RUN: llc -march=amdgcn -mattr=+fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=SI-DENORM %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+fp16-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=VI-DENORM %s + +declare half @llvm.fmuladd.f16(half %a, half %b, half %c) +declare <2 x half> @llvm.fmuladd.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c) + +; GCN-LABEL: {{^}}fmuladd_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_mac_f32_e32 v[[C_F32]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[C_F32]] +; SI: buffer_store_short v[[R_F16]] + +; VI-FLUSH: v_mac_f16_e32 v[[C_F16]], v[[B_F16]], v[[A_F16]] +; VI-FLUSH: buffer_store_short v[[C_F16]] + +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], v[[A_F16]], v[[B_F16]], v[[C_F16]] +; VI-DENORM: buffer_store_short [[RESULT]] + +; GCN: s_endpgm +define void @fmuladd_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) { + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fmuladd.f16(half %a.val, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmuladd_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_mac_f32_e32 v[[C_F32]], v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[C_F32]] +; SI: buffer_store_short v[[R_F16]] + +; VI-FLUSH: v_mac_f16_e32 v[[C_F16]], 0x4200, v[[B_F16]] +; VI-FLUSH: buffer_store_short v[[C_F16]] + +; VI-DENORM: v_mov_b32_e32 [[KA:v[0-9]+]], 0x4200 +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[KA]], v[[B_F16]], v[[C_F16]] +; VI-DENORM: buffer_store_short [[RESULT]] + +; GCN: s_endpgm +define void @fmuladd_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b, + half addrspace(1)* %c) { + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fmuladd.f16(half 3.0, half %b.val, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmuladd_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_mac_f32_e32 v[[C_F32]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[C_F32]] +; SI: buffer_store_short v[[R_F16]] + +; VI-FLUSH: v_mac_f16_e32 v[[C_F16]], 0x4200, v[[A_F16]] +; VI-FLUSH: buffer_store_short v[[C_F16]] + +; VI-DENORM: v_mov_b32_e32 [[KA:v[0-9]+]], 0x4200 +; VI-DENORM: v_fma_f16 [[RESULT:v[0-9]+]], [[KA]], v[[A_F16]], v[[C_F16]] +; VI-DENORM buffer_store_short [[RESULT]] + + +; GCN: s_endpgm +define void @fmuladd_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %c) { + %a.val = load half, half addrspace(1)* %a + %c.val = load half, half addrspace(1)* %c + %r.val = call half @llvm.fmuladd.f16(half %a.val, half 3.0, half %c.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}fmuladd_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]] +; SI: v_mac_f32_e32 v[[C_F32_0]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[C_F32_0]] +; SI: v_mac_f32_e32 v[[C_F32_1]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[C_F32_1]] +; SI: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; SI: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] + + +; FIXME: and should be unnecessary +; VI-FLUSH: v_mac_f16_e32 v[[C_V2_F16]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI-FLUSH: v_mac_f16_e32 v[[C_F16_1]], v[[B_F16_1]], v[[A_F16_1]] +; VI-FLUSH: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[C_V2_F16]] +; VI-FLUSH: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[C_F16_1]] + +; VI-DENORM-DAG: v_fma_f16 v[[RES0:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]], v[[C_V2_F16]] +; VI-DENORM-DAG: v_fma_f16 v[[RES1:[0-9]+]], v[[A_F16_1]], v[[B_F16_1]], v[[C_F16_1]] +; VI-DENORM: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[RES0]] +; VI-DENORM: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[RES1]] + +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @fmuladd_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) { + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %r.val = call <2 x half> @llvm.fmuladd.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> %c.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.log2.f16.ll b/test/CodeGen/AMDGPU/llvm.log2.f16.ll new file mode 100644 index 000000000000..8c35412b1686 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.log2.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.log2.f16(half %a) +declare <2 x half> @llvm.log2.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}log2_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_log_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_log_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @log2_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.log2.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}log2_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_log_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @log2_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.log2.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll b/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll new file mode 100644 index 000000000000..0f75f7a5a492 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.maxnum.f16.ll @@ -0,0 +1,153 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.maxnum.f16(half %a, half %b) +declare <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> %b) + +; GCN-LABEL: {{^}}maxnum_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_max_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_max_f16_e32 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @maxnum_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.maxnum.f16(half %a.val, half %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}maxnum_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_max_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_max_f16_e32 v[[R_F16:[0-9]+]], 0x4200, v[[B_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @maxnum_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.maxnum.f16(half 3.0, half %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}maxnum_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_max_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_max_f16_e32 v[[R_F16:[0-9]+]], 4.0, v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @maxnum_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.maxnum.f16(half %a.val, half 4.0) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}maxnum_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_max_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_max_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_max_f16_e32 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_max_f16_e32 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @maxnum_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a.val, <2 x half> %b.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}maxnum_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_max_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_max_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_max_f16_e32 v[[R_F16_0:[0-9]+]], 0x4200, v[[B_V2_F16]] +; VI: v_max_f16_e32 v[[R_F16_1:[0-9]+]], 4.0, v[[B_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @maxnum_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = call <2 x half> @llvm.maxnum.v2f16(<2 x half> <half 3.0, half 4.0>, <2 x half> %b.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}maxnum_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_max_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_max_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_max_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]] +; VI: v_max_f16_e32 v[[R_F16_1:[0-9]+]], 0x4200, v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @maxnum_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.maxnum.v2f16(<2 x half> %a.val, <2 x half> <half 4.0, half 3.0>) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.memcpy.ll b/test/CodeGen/AMDGPU/llvm.memcpy.ll index 8398309d7520..ccb383e0d027 100644 --- a/test/CodeGen/AMDGPU/llvm.memcpy.ll +++ b/test/CodeGen/AMDGPU/llvm.memcpy.ll @@ -1,8 +1,9 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s declare void @llvm.memcpy.p3i8.p3i8.i32(i8 addrspace(3)* nocapture, i8 addrspace(3)* nocapture, i32, i32, i1) nounwind declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(1)* nocapture, i64, i32, i1) nounwind +declare void @llvm.memcpy.p1i8.p2i8.i64(i8 addrspace(1)* nocapture, i8 addrspace(2)* nocapture, i64, i32, i1) nounwind ; FUNC-LABEL: {{^}}test_small_memcpy_i64_lds_to_lds_align1: @@ -325,3 +326,47 @@ define void @test_small_memcpy_i64_global_to_global_align16(i64 addrspace(1)* no call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %bcout, i8 addrspace(1)* %bcin, i64 32, i32 16, i1 false) nounwind ret void } + +; Test shouldConvertConstantLoadToIntImm +@hello.align4 = private unnamed_addr addrspace(2) constant [16 x i8] c"constant string\00", align 4 +@hello.align1 = private unnamed_addr addrspace(2) constant [16 x i8] c"constant string\00", align 1 + +; FUNC-LABEL: {{^}}test_memcpy_const_string_align4: +; SI: s_getpc_b64 +; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, hello.align4+20 +; SI: s_addc_u32 +; SI-DAG: s_load_dwordx4 +; SI-DAG: s_load_dwordx4 +; SI-DAG: s_load_dwordx2 +; SI-DAG: buffer_store_dwordx4 +; SI-DAG: buffer_store_dwordx4 +define void @test_memcpy_const_string_align4(i8 addrspace(1)* noalias %out) nounwind { + %str = bitcast [16 x i8] addrspace(2)* @hello.align4 to i8 addrspace(2)* + call void @llvm.memcpy.p1i8.p2i8.i64(i8 addrspace(1)* %out, i8 addrspace(2)* %str, i64 32, i32 4, i1 false) + ret void +} + +; FUNC-LABEL: {{^}}test_memcpy_const_string_align1: +; SI-NOT: buffer_load +; SI: v_mov_b32_e32 v{{[0-9]+}}, 0x69 +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +define void @test_memcpy_const_string_align1(i8 addrspace(1)* noalias %out) nounwind { + %str = bitcast [16 x i8] addrspace(2)* @hello.align1 to i8 addrspace(2)* + call void @llvm.memcpy.p1i8.p2i8.i64(i8 addrspace(1)* %out, i8 addrspace(2)* %str, i64 32, i32 1, i1 false) + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.minnum.f16.ll b/test/CodeGen/AMDGPU/llvm.minnum.f16.ll new file mode 100644 index 000000000000..6bf2e9ba2e32 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.minnum.f16.ll @@ -0,0 +1,153 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.minnum.f16(half %a, half %b) +declare <2 x half> @llvm.minnum.v2f16(<2 x half> %a, <2 x half> %b) + +; GCN-LABEL: {{^}}minnum_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_min_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_min_f16_e32 v[[R_F16:[0-9]+]], v[[B_F16]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @minnum_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.minnum.f16(half %a.val, half %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}minnum_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_min_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_min_f16_e32 v[[R_F16:[0-9]+]], 0x4200, v[[B_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @minnum_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b) { +entry: + %b.val = load half, half addrspace(1)* %b + %r.val = call half @llvm.minnum.f16(half 3.0, half %b.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}minnum_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_min_f32_e32 v[[R_F32:[0-9]+]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_min_f16_e32 v[[R_F16:[0-9]+]], 4.0, v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @minnum_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.minnum.f16(half %a.val, half 4.0) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}minnum_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_min_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_min_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_min_f16_e32 v[[R_F16_0:[0-9]+]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_min_f16_e32 v[[R_F16_1:[0-9]+]], v[[B_F16_1]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @minnum_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = call <2 x half> @llvm.minnum.v2f16(<2 x half> %a.val, <2 x half> %b.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}minnum_v2f16_imm_a +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_min_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_min_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_min_f16_e32 v[[R_F16_0:[0-9]+]], 0x4200, v[[B_V2_F16]] +; VI: v_min_f16_e32 v[[R_F16_1:[0-9]+]], 4.0, v[[B_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @minnum_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %r.val = call <2 x half> @llvm.minnum.v2f16(<2 x half> <half 3.0, half 4.0>, <2 x half> %b.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}minnum_v2f16_imm_b +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], 0x4400{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], 0x4200{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_min_f32_e32 v[[R_F32_0:[0-9]+]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_min_f32_e32 v[[R_F32_1:[0-9]+]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_min_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]] +; VI: v_min_f16_e32 v[[R_F16_1:[0-9]+]], 0x4200, v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @minnum_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.minnum.v2f16(<2 x half> %a.val, <2 x half> <half 4.0, half 3.0>) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.r600.read.workdim.ll b/test/CodeGen/AMDGPU/llvm.r600.read.workdim.ll deleted file mode 100644 index 2f5947395c43..000000000000 --- a/test/CodeGen/AMDGPU/llvm.r600.read.workdim.ll +++ /dev/null @@ -1,36 +0,0 @@ -; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG %s - -; EG-LABEL: {{^}}read_workdim: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV * [[VAL]], KC0[2].Z -define void @read_workdim(i32 addrspace(1)* %out) { -entry: - %dim = call i32 @llvm.r600.read.workdim() #0 - store i32 %dim, i32 addrspace(1)* %out - ret void -} - -; EG-LABEL: {{^}}read_workdim_known_bits: -define void @read_workdim_known_bits(i32 addrspace(1)* %out) { -entry: - %dim = call i32 @llvm.r600.read.workdim() #0 - %shl = shl i32 %dim, 24 - %shr = lshr i32 %shl, 24 - store i32 %shr, i32 addrspace(1)* %out - ret void -} - -; EG-LABEL: {{^}}legacy_read_workdim: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV * [[VAL]], KC0[2].Z -define void @legacy_read_workdim(i32 addrspace(1)* %out) { -entry: - %dim = call i32 @llvm.AMDGPU.read.workdim() #0 - store i32 %dim, i32 addrspace(1)* %out - ret void -} - -declare i32 @llvm.r600.read.workdim() #0 -declare i32 @llvm.AMDGPU.read.workdim() #0 - -attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/llvm.rint.f16.ll b/test/CodeGen/AMDGPU/llvm.rint.f16.ll new file mode 100644 index 000000000000..ad9b66c9038c --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.rint.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.rint.f16(half %a) +declare <2 x half> @llvm.rint.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}rint_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_rndne_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_rndne_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @rint_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.rint.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}rint_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_rndne_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_rndne_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_rndne_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_rndne_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @rint_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.rint.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.round.f64.ll b/test/CodeGen/AMDGPU/llvm.round.f64.ll index a8024b713261..3ea4551f0ee7 100644 --- a/test/CodeGen/AMDGPU/llvm.round.f64.ll +++ b/test/CodeGen/AMDGPU/llvm.round.f64.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}round_f64: ; SI: s_endpgm @@ -18,9 +18,9 @@ define void @round_f64(double addrspace(1)* %out, double %x) #0 { ; SI-DAG: v_not_b32_e32 ; SI-DAG: v_not_b32_e32 -; SI-DAG: v_cmp_eq_i32 +; SI-DAG: v_cmp_eq_u32 -; SI-DAG: s_mov_b32 [[BFIMASK:s[0-9]+]], 0x7fffffff +; SI-DAG: s_brev_b32 [[BFIMASK:s[0-9]+]], -2{{$}} ; SI-DAG: v_cmp_gt_i32 ; SI-DAG: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[BFIMASK]] diff --git a/test/CodeGen/AMDGPU/llvm.round.ll b/test/CodeGen/AMDGPU/llvm.round.ll index 9b7bb00d3c38..86002662e0b7 100644 --- a/test/CodeGen/AMDGPU/llvm.round.ll +++ b/test/CodeGen/AMDGPU/llvm.round.ll @@ -1,15 +1,15 @@ -; RUN: llc -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}round_f32: ; SI-DAG: s_load_dword [[SX:s[0-9]+]] -; SI-DAG: s_mov_b32 [[K:s[0-9]+]], 0x7fffffff +; SI-DAG: s_brev_b32 [[K:s[0-9]+]], -2{{$}} ; SI-DAG: v_trunc_f32_e32 [[TRUNC:v[0-9]+]], [[SX]] ; SI-DAG: v_sub_f32_e32 [[SUB:v[0-9]+]], [[SX]], [[TRUNC]] ; SI-DAG: v_mov_b32_e32 [[VX:v[0-9]+]], [[SX]] ; SI: v_bfi_b32 [[COPYSIGN:v[0-9]+]], [[K]], 1.0, [[VX]] -; SI: v_cmp_le_f32_e64 vcc, 0.5, |[[SUB]]| +; SI: v_cmp_ge_f32_e64 vcc, |[[SUB]]|, 0.5 ; SI: v_cndmask_b32_e32 [[SEL:v[0-9]+]], 0, [[VX]] ; SI: v_add_f32_e32 [[RESULT:v[0-9]+]], [[SEL]], [[TRUNC]] ; SI: buffer_store_dword [[RESULT]] diff --git a/test/CodeGen/AMDGPU/llvm.sin.f16.ll b/test/CodeGen/AMDGPU/llvm.sin.f16.ll new file mode 100644 index 000000000000..8374a75370b2 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.sin.f16.ll @@ -0,0 +1,55 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.sin.f16(half %a) +declare <2 x half> @llvm.sin.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}sin_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; GCN: v_mul_f32_e32 v[[M_F32:[0-9]+]], {{0.15915494|0x3e22f983}}, v[[A_F32]] +; GCN: v_fract_f32_e32 v[[F_F32:[0-9]+]], v[[M_F32]] +; GCN: v_sin_f32_e32 v[[R_F32:[0-9]+]], v[[F_F32]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @sin_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.sin.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}sin_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_mov_b32_e32 v[[HALF_PIE:[0-9]+]], 0x3e22f983{{$}} +; GCN: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_mul_f32_e32 v[[M_F32_0:[0-9]+]], v[[HALF_PIE]], v[[A_F32_0]] +; VI: v_mul_f32_e32 v[[M_F32_0:[0-9]+]], 0.15915494, v[[A_F32_0]] +; GCN: v_fract_f32_e32 v[[F_F32_0:[0-9]+]], v[[M_F32_0]] +; SI: v_mul_f32_e32 v[[M_F32_1:[0-9]+]], v[[HALF_PIE]], v[[A_F32_1]] +; VI: v_mul_f32_e32 v[[M_F32_1:[0-9]+]], 0.15915494, v[[A_F32_1]] +; GCN: v_fract_f32_e32 v[[F_F32_1:[0-9]+]], v[[M_F32_1]] +; GCN: v_sin_f32_e32 v[[R_F32_0:[0-9]+]], v[[F_F32_0]] +; GCN: v_sin_f32_e32 v[[R_F32_1:[0-9]+]], v[[F_F32_1]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @sin_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.sin.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll new file mode 100644 index 000000000000..8217ebf46738 --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.sqrt.f16(half %a) +declare <2 x half> @llvm.sqrt.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}sqrt_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_sqrt_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_sqrt_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @sqrt_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.sqrt.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}sqrt_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_sqrt_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_sqrt_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_sqrt_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_sqrt_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @sqrt_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.sqrt.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/llvm.trunc.f16.ll b/test/CodeGen/AMDGPU/llvm.trunc.f16.ll new file mode 100644 index 000000000000..04054c2b70fe --- /dev/null +++ b/test/CodeGen/AMDGPU/llvm.trunc.f16.ll @@ -0,0 +1,49 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +declare half @llvm.trunc.f16(half %a) +declare <2 x half> @llvm.trunc.v2f16(<2 x half> %a) + +; GCN-LABEL: {{^}}trunc_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_trunc_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_trunc_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @trunc_f16( + half addrspace(1)* %r, + half addrspace(1)* %a) { +entry: + %a.val = load half, half addrspace(1)* %a + %r.val = call half @llvm.trunc.f16(half %a.val) + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}trunc_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_trunc_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]] +; SI: v_trunc_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]] +; VI: v_trunc_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_V2_F16]] +; VI: v_trunc_f16_e32 v[[R_F16_1:[0-9]+]], v[[A_F16_1]] +; GCN: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; GCN: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @trunc_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %r.val = call <2 x half> @llvm.trunc.v2f16(<2 x half> %a.val) + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/load-constant-i16.ll b/test/CodeGen/AMDGPU/load-constant-i16.ll index ef9791d8f7a1..628d285141bc 100644 --- a/test/CodeGen/AMDGPU/load-constant-i16.ll +++ b/test/CodeGen/AMDGPU/load-constant-i16.ll @@ -1,13 +1,13 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,GCN-NOHSA-SI,FUNC %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,GCN-NOHSA-VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}constant_load_i16: ; GCN-NOHSA: buffer_load_ushort v{{[0-9]+}} ; GCN-HSA: flat_load_ushort -; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0 +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_load_i16(i16 addrspace(1)* %out, i16 addrspace(2)* %in) { entry: %ld = load i16, i16 addrspace(2)* %in @@ -18,7 +18,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v2i16: ; GCN: s_load_dword s -; EG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(2)* %in) { entry: %ld = load <2 x i16>, <2 x i16> addrspace(2)* %in @@ -29,8 +29,8 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v3i16: ; GCN: s_load_dwordx2 s -; EG-DAG: VTX_READ_32 -; EG-DAG: VTX_READ_16 +; EG-DAG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 4, #1 define void @constant_load_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> addrspace(2)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(2)* %in @@ -41,7 +41,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v4i16: ; GCN: s_load_dwordx2 -; EG: VTX_READ_64 +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(2)* %in) { entry: %ld = load <4 x i16>, <4 x i16> addrspace(2)* %in @@ -52,7 +52,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v8i16: ; GCN: s_load_dwordx4 -; EG: VTX_READ_128 +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(2)* %in) { entry: %ld = load <8 x i16>, <8 x i16> addrspace(2)* %in @@ -63,8 +63,8 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v16i16: ; GCN: s_load_dwordx8 -; EG: VTX_READ_128 -; EG: VTX_READ_128 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_load_v16i16(<16 x i16> addrspace(1)* %out, <16 x i16> addrspace(2)* %in) { entry: %ld = load <16 x i16>, <16 x i16> addrspace(2)* %in @@ -79,7 +79,7 @@ entry: ; GCN-HSA: flat_load_ushort ; GCN-HSA: flat_store_dword -; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}} +; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}, 0, #1 define void @constant_zextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(2)* %in) #0 { %a = load i16, i16 addrspace(2)* %in %ext = zext i16 %a to i32 @@ -94,7 +94,7 @@ define void @constant_zextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace( ; GCN-HSA: flat_load_sshort ; GCN-HSA: flat_store_dword -; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]] +; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal ; EG: 16 define void @constant_sextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(2)* %in) #0 { @@ -107,6 +107,8 @@ define void @constant_sextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace( ; FUNC-LABEL: {{^}}constant_zextload_v1i16_to_v1i32: ; GCN-NOHSA: buffer_load_ushort ; GCN-HSA: flat_load_ushort + +; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}}, 0, #1 define void @constant_zextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(2)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(2)* %in %ext = zext <1 x i16> %load to <1 x i32> @@ -117,6 +119,10 @@ define void @constant_zextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x ; FUNC-LABEL: {{^}}constant_sextload_v1i16_to_v1i32: ; GCN-NOHSA: buffer_load_sshort ; GCN-HSA: flat_load_sshort + +; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal +; EG: 16 define void @constant_sextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(2)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(2)* %in %ext = sext <1 x i16> %load to <1 x i32> @@ -128,6 +134,12 @@ define void @constant_sextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x ; GCN: s_load_dword s ; GCN-DAG: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0xffff{{$}} ; GCN-DAG: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 + +; v2i16 is naturally 4 byte aligned +; EG: VTX_READ_32 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; TODO: This should use DST, but for some there are redundant MOVs +; EG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, literal +; EG: 16 define void @constant_zextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(2)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(2)* %in %ext = zext <2 x i16> %load to <2 x i32> @@ -140,10 +152,12 @@ define void @constant_zextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i16 -; EG-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal +; v2i16 is naturally 4 byte aligned +; EG: VTX_READ_32 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; TODO: We should also use ASHR instead of LSHR + BFE +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, 0.0, literal ; EG-DAG: 16 ; EG-DAG: 16 define void @constant_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(2)* %in) #0 { @@ -155,6 +169,13 @@ define void @constant_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x ; FUNC-LABEL: {{^}}constant_constant_zextload_v3i16_to_v3i32: ; GCN: s_load_dwordx2 + +; v3i16 is naturally 8 byte aligned +; EG-DAG: VTX_READ_32 [[DST_HI:T[0-9]\.[XYZW]]], [[DST_HI]], 0, #1 +; EG-DAG: VTX_READ_16 [[DST_LO:T[0-9]\.[XYZW]]], [[DST_LO]], 4, #1 +; TODO: This should use DST, but for some there are redundant MOVs +; EG: LSHR {{[* ]*}}{{T[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG: 16 define void @constant_constant_zextload_v3i16_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i16> addrspace(2)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(2)* %in @@ -165,6 +186,16 @@ entry: ; FUNC-LABEL: {{^}}constant_constant_sextload_v3i16_to_v3i32: ; GCN: s_load_dwordx2 + +; v3i16 is naturally 8 byte aligned +; EG-DAG: VTX_READ_32 [[DST_HI:T[0-9]\.[XYZW]]], [[DST_HI]], 0, #1 +; EG-DAG: VTX_READ_16 [[DST_LO:T[0-9]\.[XYZW]]], [[DST_LO]], 4, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: ASHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, 0.0, literal +; EG-DAG: 16 +; EG-DAG: 16 define void @constant_constant_sextload_v3i16_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i16> addrspace(2)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(2)* %in @@ -178,10 +209,14 @@ entry: ; GCN-DAG: s_and_b32 ; GCN-DAG: s_lshr_b32 -; EG: VTX_READ_16 -; EG: VTX_READ_16 -; EG: VTX_READ_16 -; EG: VTX_READ_16 +; v4i16 is naturally 8 byte aligned +; EG: VTX_READ_64 [[DST:T[0-9]\.XY]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, literal +; EG-DAG: 16 +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG-DAG: AND_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG-DAG: 16 define void @constant_constant_zextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(2)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(2)* %in %ext = zext <4 x i16> %load to <4 x i32> @@ -194,14 +229,14 @@ define void @constant_constant_zextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* % ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i16 -; EG-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: VTX_READ_16 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]] -; EG-DAG: VTX_READ_16 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal +; v4i16 is naturally 8 byte aligned +; EG: VTX_READ_64 [[DST:T[0-9]\.XY]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; TODO: We should use ASHR instead of LSHR + BFE +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 16 ; EG-DAG: 16 ; EG-DAG: 16 @@ -217,6 +252,26 @@ define void @constant_sextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x ; GCN: s_load_dwordx4 ; GCN-DAG: s_and_b32 ; GCN-DAG: s_lshr_b32 + +; v8i16 is naturally 16 byte aligned +; EG: VTX_READ_128 [[DST:T[0-9]\.XYZW]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 define void @constant_zextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(2)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(2)* %in %ext = zext <8 x i16> %load to <8 x i32> @@ -228,6 +283,27 @@ define void @constant_zextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x ; GCN: s_load_dwordx4 ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i16 + +; v8i16 is naturally 16 byte aligned +; EG: VTX_READ_128 [[DST:T[0-9]\.XYZW]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; TODO: We should use ASHR instead of LSHR + BFE +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 define void @constant_sextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(2)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(2)* %in %ext = sext <8 x i16> %load to <8 x i32> @@ -239,6 +315,10 @@ define void @constant_sextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x ; GCN: s_load_dwordx8 ; GCN-DAG: s_and_b32 ; GCN-DAG: s_lshr_b32 + +; v16i16 is naturally 32 byte aligned +; EG-DAG: VTX_READ_128 [[DST_HI:T[0-9]+\.XYZW]], {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 [[DST_LO:T[0-9]+\.XYZW]], {{T[0-9]+.[XYZW]}}, 16, #1 define void @constant_zextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(2)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(2)* %in %ext = zext <16 x i16> %load to <16 x i32> @@ -250,6 +330,10 @@ define void @constant_zextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, < ; GCN: s_load_dwordx8 ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i16 + +; v16i16 is naturally 32 byte aligned +; EG-DAG: VTX_READ_128 [[DST_HI:T[0-9]+\.XYZW]], {{T[0-9]+\.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 [[DST_LO:T[0-9]+\.XYZW]], {{T[0-9]+\.[XYZW]}}, 16, #1 define void @constant_sextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(2)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(2)* %in %ext = sext <16 x i16> %load to <16 x i32> @@ -260,8 +344,13 @@ define void @constant_sextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, < ; FUNC-LABEL: {{^}}constant_zextload_v32i16_to_v32i32: ; GCN-DAG: s_load_dwordx16 ; GCN-DAG: s_mov_b32 [[K:s[0-9]+]], 0xffff{{$}} -; GCN: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[K]] -; GCN: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 +; GCN-DAG: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 +; GCN-DAG: s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[K]] + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 48, #1 define void @constant_zextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(2)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(2)* %in %ext = zext <32 x i16> %load to <32 x i32> @@ -273,6 +362,11 @@ define void @constant_zextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, < ; GCN: s_load_dwordx16 ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i16 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 48, #1 define void @constant_sextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(2)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(2)* %in %ext = sext <32 x i16> %load to <32 x i32> @@ -283,6 +377,15 @@ define void @constant_sextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, < ; FUNC-LABEL: {{^}}constant_zextload_v64i16_to_v64i32: ; GCN: s_load_dwordx16 ; GCN: s_load_dwordx16 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 48, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 64, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 80, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 96, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 112, #1 define void @constant_zextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(2)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(2)* %in %ext = zext <64 x i16> %load to <64 x i32> @@ -291,6 +394,15 @@ define void @constant_zextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, < } ; FUNC-LABEL: {{^}}constant_sextload_v64i16_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 48, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 64, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 80, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 96, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+\.[XYZW]}}, 112, #1 define void @constant_sextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(2)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(2)* %in %ext = sext <64 x i16> %load to <64 x i32> @@ -305,6 +417,9 @@ define void @constant_sextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, < ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @constant_zextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(2)* %in) #0 { %a = load i16, i16 addrspace(2)* %in %ext = zext i16 %a to i64 @@ -313,12 +428,24 @@ define void @constant_zextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace( } ; FUNC-LABEL: {{^}}constant_sextload_i16_to_i64: -; GCN-NOHSA-DAG: buffer_load_sshort v[[LO:[0-9]+]], +; FIXME: Need to optimize this sequence to avoid extra bfe: +; t28: i32,ch = load<LD2[%in(addrspace=1)], anyext from i16> t12, t27, undef:i64 +; t31: i64 = any_extend t28 +; t33: i64 = sign_extend_inreg t31, ValueType:ch:i16 + +; GCN-NOHSA-SI-DAG: buffer_load_sshort v[[LO:[0-9]+]], ; GCN-HSA-DAG: flat_load_sshort v[[LO:[0-9]+]], +; GCN-NOHSA-VI-DAG: buffer_load_ushort v[[ULO:[0-9]+]], +; GCN-NOHSA-VI-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16 ; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 15 ? +; EG: 31 define void @constant_sextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(2)* %in) #0 { %a = load i16, i16 addrspace(2)* %in %ext = sext i16 %a to i64 @@ -327,6 +454,9 @@ define void @constant_sextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace( } ; FUNC-LABEL: {{^}}constant_zextload_v1i16_to_v1i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @constant_zextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(2)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(2)* %in %ext = zext <1 x i16> %load to <1 x i64> @@ -335,6 +465,11 @@ define void @constant_zextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_sextload_v1i16_to_v1i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 15 ? +; EG: 31 define void @constant_sextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(2)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(2)* %in %ext = sext <1 x i16> %load to <1 x i64> @@ -343,6 +478,8 @@ define void @constant_sextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_zextload_v2i16_to_v2i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i16> addrspace(2)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(2)* %in %ext = zext <2 x i16> %load to <2 x i64> @@ -351,6 +488,8 @@ define void @constant_zextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_sextload_v2i16_to_v2i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i16> addrspace(2)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(2)* %in %ext = sext <2 x i16> %load to <2 x i64> @@ -359,6 +498,8 @@ define void @constant_sextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_zextload_v4i16_to_v4i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(2)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(2)* %in %ext = zext <4 x i16> %load to <4 x i64> @@ -367,6 +508,8 @@ define void @constant_zextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_sextload_v4i16_to_v4i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(2)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(2)* %in %ext = sext <4 x i16> %load to <4 x i64> @@ -375,6 +518,8 @@ define void @constant_sextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_zextload_v8i16_to_v8i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(2)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(2)* %in %ext = zext <8 x i16> %load to <8 x i64> @@ -383,6 +528,8 @@ define void @constant_zextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_sextload_v8i16_to_v8i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(2)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(2)* %in %ext = sext <8 x i16> %load to <8 x i64> @@ -391,6 +538,9 @@ define void @constant_sextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_zextload_v16i16_to_v16i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_zextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(2)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(2)* %in %ext = zext <16 x i16> %load to <16 x i64> @@ -399,6 +549,9 @@ define void @constant_zextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, < } ; FUNC-LABEL: {{^}}constant_sextload_v16i16_to_v16i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_sextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(2)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(2)* %in %ext = sext <16 x i16> %load to <16 x i64> @@ -407,6 +560,11 @@ define void @constant_sextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, < } ; FUNC-LABEL: {{^}}constant_zextload_v32i16_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 48, #1 define void @constant_zextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(2)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(2)* %in %ext = zext <32 x i16> %load to <32 x i64> @@ -415,6 +573,11 @@ define void @constant_zextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, < } ; FUNC-LABEL: {{^}}constant_sextload_v32i16_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 48, #1 define void @constant_sextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(2)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(2)* %in %ext = sext <32 x i16> %load to <32 x i64> @@ -422,6 +585,8 @@ define void @constant_sextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, < ret void } +; These trigger undefined register machine verifier errors + ; ; XFUNC-LABEL: {{^}}constant_zextload_v64i16_to_v64i64: ; define void @constant_zextload_v64i16_to_v64i64(<64 x i64> addrspace(1)* %out, <64 x i16> addrspace(2)* %in) #0 { ; %load = load <64 x i16>, <64 x i16> addrspace(2)* %in diff --git a/test/CodeGen/AMDGPU/load-constant-i32.ll b/test/CodeGen/AMDGPU/load-constant-i32.ll index 40c29be60548..56b9e3a187c6 100644 --- a/test/CodeGen/AMDGPU/load-constant-i32.ll +++ b/test/CodeGen/AMDGPU/load-constant-i32.ll @@ -277,47 +277,47 @@ define void @constant_zextload_v16i32_to_v16i64(<16 x i64> addrspace(1)* %out, < ; FUNC-LABEL: {{^}}constant_sextload_v32i32_to_v32i64: ; GCN: s_load_dwordx16 -; GCN: s_load_dwordx16 +; GCN-DAG: s_load_dwordx16 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 -; GCN-NOHSA: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 +; GCN-NOHSA-DAG: buffer_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 -; GCN-HSA: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 +; GCN-HSA-DAG: flat_store_dwordx4 define void @constant_sextload_v32i32_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i32> addrspace(2)* %in) #0 { %ld = load <32 x i32>, <32 x i32> addrspace(2)* %in diff --git a/test/CodeGen/AMDGPU/load-constant-i8.ll b/test/CodeGen/AMDGPU/load-constant-i8.ll index 87828982a987..39a299fcd327 100644 --- a/test/CodeGen/AMDGPU/load-constant-i8.ll +++ b/test/CodeGen/AMDGPU/load-constant-i8.ll @@ -8,7 +8,8 @@ ; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}} ; GCN-HSA: flat_load_ubyte -; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0 +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: NOT AND define void @constant_load_i8(i8 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { entry: %ld = load i8, i8 addrspace(2)* %in @@ -20,7 +21,7 @@ entry: ; GCN-NOHSA: buffer_load_ushort v ; GCN-HSA: flat_load_ushort v -; EG: VTX_READ_16 +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { entry: %ld = load <2 x i8>, <2 x i8> addrspace(2)* %in @@ -31,7 +32,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v3i8: ; GCN: s_load_dword s -; EG-DAG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v3i8(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(2)* %in @@ -42,7 +43,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v4i8: ; GCN: s_load_dword s -; EG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { entry: %ld = load <4 x i8>, <4 x i8> addrspace(2)* %in @@ -53,7 +54,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v8i8: ; GCN: s_load_dwordx2 -; EG: VTX_READ_64 +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { entry: %ld = load <8 x i8>, <8 x i8> addrspace(2)* %in @@ -64,7 +65,7 @@ entry: ; FUNC-LABEL: {{^}}constant_load_v16i8: ; GCN: s_load_dwordx4 -; EG: VTX_READ_128 +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_load_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { entry: %ld = load <16 x i8>, <16 x i8> addrspace(2)* %in @@ -76,7 +77,7 @@ entry: ; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}}, ; GCN-HSA: flat_load_ubyte -; EG: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}} +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { %a = load i8, i8 addrspace(2)* %in %ext = zext i8 %a to i32 @@ -88,7 +89,7 @@ define void @constant_zextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(2) ; GCN-NOHSA: buffer_load_sbyte ; GCN-HSA: flat_load_sbyte -; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]] +; EG: VTX_READ_8 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal ; EG: 8 define void @constant_sextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { @@ -99,6 +100,8 @@ define void @constant_sextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(2) } ; FUNC-LABEL: {{^}}constant_zextload_v1i8_to_v1i32: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8> addrspace(2)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(2)* %in %ext = zext <1 x i8> %load to <1 x i32> @@ -107,6 +110,10 @@ define void @constant_zextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_sextload_v1i8_to_v1i32: + +; EG: VTX_READ_8 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal +; EG: 8 define void @constant_sextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8> addrspace(2)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(2)* %in %ext = sext <1 x i8> %load to <1 x i32> @@ -117,8 +124,11 @@ define void @constant_sextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x ; FUNC-LABEL: {{^}}constant_zextload_v2i8_to_v2i32: ; GCN-NOHSA: buffer_load_ushort ; GCN-HSA: flat_load_ushort -; EG: VTX_READ_8 -; EG: VTX_READ_8 + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: This should use DST, but for some there are redundant MOVs +; EG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG: 8 define void @constant_zextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(2)* %in %ext = zext <2 x i8> %load to <2 x i32> @@ -134,10 +144,10 @@ define void @constant_zextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x ; GCN: v_bfe_i32 ; GCN: v_bfe_i32 -; EG-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal +; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 8 ; EG-DAG: 8 define void @constant_sextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { @@ -153,6 +163,13 @@ define void @constant_sextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x ; GCN-DAG: s_bfe_u32 ; GCN-DAG: s_bfe_u32 ; GCN-DAG: s_and_b32 + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_zextload_v3i8_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(2)* %in @@ -167,6 +184,15 @@ entry: ; GCN-DAG: s_bfe_i32 ; GCN-DAG: s_bfe_i32 ; GCN-DAG: s_bfe_i32 + +; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_sextload_v3i8_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i8> addrspace(2)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(2)* %in @@ -180,10 +206,14 @@ entry: ; GCN-DAG: s_and_b32 ; GCN-DAG: s_lshr_b32 -; EG: VTX_READ_8 -; EG: VTX_READ_8 -; EG: VTX_READ_8 -; EG: VTX_READ_8 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_zextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(2)* %in %ext = zext <4 x i8> %load to <4 x i32> @@ -196,14 +226,12 @@ define void @constant_zextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x ; GCN-DAG: s_sext_i32_i8 ; GCN-DAG: s_ashr_i32 -; EG-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: VTX_READ_8 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]] -; EG-DAG: VTX_READ_8 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal +; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 8 ; EG-DAG: 8 ; EG-DAG: 8 @@ -219,6 +247,23 @@ define void @constant_sextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x ; GCN: s_load_dwordx2 ; GCN-DAG: s_and_b32 ; GCN-DAG: s_lshr_b32 + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_zextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = zext <8 x i8> %load to <8 x i32> @@ -230,6 +275,25 @@ define void @constant_zextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x ; GCN: s_load_dwordx2 ; GCN-DAG: s_ashr_i32 ; GCN-DAG: s_sext_i32_i8 + +; EG: VTX_READ_64 [[DST:T[0-9]+\.XY]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_sextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = sext <8 x i8> %load to <8 x i32> @@ -238,6 +302,39 @@ define void @constant_sextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_zextload_v16i8_to_v16i32: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_zextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = zext <16 x i8> %load to <16 x i32> @@ -246,6 +343,41 @@ define void @constant_zextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_sextload_v16i8_to_v16i32: + +; EG: VTX_READ_128 [[DST:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_sextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = sext <16 x i8> %load to <16 x i32> @@ -254,6 +386,70 @@ define void @constant_sextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_zextload_v32i8_to_v32i32: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_zextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = zext <32 x i8> %load to <32 x i32> @@ -262,6 +458,74 @@ define void @constant_zextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <3 } ; FUNC-LABEL: {{^}}constant_sextload_v32i8_to_v32i32: + +; EG-DAG: VTX_READ_128 [[DST_LO:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 [[DST_HI:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 16, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @constant_sextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = sext <32 x i8> %load to <32 x i32> @@ -270,6 +534,11 @@ define void @constant_sextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <3 } ; FUNC-LABEL: {{^}}constant_zextload_v64i8_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1 define void @constant_zextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i8> addrspace(2)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(2)* %in %ext = zext <64 x i8> %load to <64 x i32> @@ -278,6 +547,11 @@ define void @constant_zextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <6 } ; FUNC-LABEL: {{^}}constant_sextload_v64i8_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1 define void @constant_sextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i8> addrspace(2)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(2)* %in %ext = sext <64 x i8> %load to <64 x i32> @@ -293,6 +567,9 @@ define void @constant_sextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <6 ; GCN-HSA-DAG: flat_load_ubyte v[[LO:[0-9]+]], ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]] + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @constant_zextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { %a = load i8, i8 addrspace(2)* %in %ext = zext i8 %a to i64 @@ -307,6 +584,11 @@ define void @constant_zextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(2) ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 7 ? +; EG: 31 define void @constant_sextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { %a = load i8, i8 addrspace(2)* %in %ext = sext i8 %a to i64 @@ -315,6 +597,9 @@ define void @constant_sextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(2) } ; FUNC-LABEL: {{^}}constant_zextload_v1i8_to_v1i64: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @constant_zextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8> addrspace(2)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(2)* %in %ext = zext <1 x i8> %load to <1 x i64> @@ -323,6 +608,11 @@ define void @constant_zextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_sextload_v1i8_to_v1i64: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 7 ? +; EG: 31 define void @constant_sextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8> addrspace(2)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(2)* %in %ext = sext <1 x i8> %load to <1 x i64> @@ -331,6 +621,8 @@ define void @constant_sextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_zextload_v2i8_to_v2i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(2)* %in %ext = zext <2 x i8> %load to <2 x i64> @@ -339,6 +631,8 @@ define void @constant_zextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_sextload_v2i8_to_v2i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(2)* %in %ext = sext <2 x i8> %load to <2 x i64> @@ -347,6 +641,8 @@ define void @constant_sextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_zextload_v4i8_to_v4i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(2)* %in %ext = zext <4 x i8> %load to <4 x i64> @@ -355,6 +651,8 @@ define void @constant_zextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_sextload_v4i8_to_v4i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(2)* %in %ext = sext <4 x i8> %load to <4 x i64> @@ -363,6 +661,8 @@ define void @constant_sextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_zextload_v8i8_to_v8i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = zext <8 x i8> %load to <8 x i64> @@ -371,6 +671,8 @@ define void @constant_zextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_sextload_v8i8_to_v8i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = sext <8 x i8> %load to <8 x i64> @@ -379,6 +681,8 @@ define void @constant_sextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_zextload_v16i8_to_v16i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = zext <16 x i8> %load to <16 x i64> @@ -387,6 +691,8 @@ define void @constant_zextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_sextload_v16i8_to_v16i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = sext <16 x i8> %load to <16 x i64> @@ -395,6 +701,9 @@ define void @constant_sextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_zextload_v32i8_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_zextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = zext <32 x i8> %load to <32 x i64> @@ -403,6 +712,9 @@ define void @constant_zextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <3 } ; FUNC-LABEL: {{^}}constant_sextload_v32i8_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_sextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = sext <32 x i8> %load to <32 x i64> @@ -445,6 +757,8 @@ define void @constant_zextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(2) ; GCN-NOHSA: buffer_store_short v[[VAL]] ; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]] + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_sextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(2)* %in) #0 { %a = load i8, i8 addrspace(2)* %in %ext = sext i8 %a to i16 @@ -461,6 +775,9 @@ define void @constant_zextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_sextload_v1i8_to_v1i16: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @constant_sextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x i8> addrspace(2)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(2)* %in %ext = sext <1 x i8> %load to <1 x i16> @@ -469,6 +786,8 @@ define void @constant_sextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x } ; FUNC-LABEL: {{^}}constant_zextload_v2i8_to_v2i16: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(2)* %in %ext = zext <2 x i8> %load to <2 x i16> @@ -477,6 +796,10 @@ define void @constant_zextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_sextload_v2i8_to_v2i16: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @constant_sextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8> addrspace(2)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(2)* %in %ext = sext <2 x i8> %load to <2 x i16> @@ -485,6 +808,8 @@ define void @constant_sextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x } ; FUNC-LABEL: {{^}}constant_zextload_v4i8_to_v4i16: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(2)* %in %ext = zext <4 x i8> %load to <4 x i16> @@ -493,6 +818,12 @@ define void @constant_zextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_sextload_v4i8_to_v4i16: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @constant_sextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8> addrspace(2)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(2)* %in %ext = sext <4 x i8> %load to <4 x i16> @@ -501,6 +832,8 @@ define void @constant_sextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}constant_zextload_v8i8_to_v8i16: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = zext <8 x i8> %load to <8 x i16> @@ -509,6 +842,17 @@ define void @constant_zextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_sextload_v8i8_to_v8i16: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal + define void @constant_sextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8> addrspace(2)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(2)* %in %ext = sext <8 x i8> %load to <8 x i16> @@ -517,6 +861,8 @@ define void @constant_sextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x } ; FUNC-LABEL: {{^}}constant_zextload_v16i8_to_v16i16: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @constant_zextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = zext <16 x i8> %load to <16 x i16> @@ -525,6 +871,24 @@ define void @constant_zextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_sextload_v16i8_to_v16i16: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @constant_sextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 x i8> addrspace(2)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(2)* %in %ext = sext <16 x i8> %load to <16 x i16> @@ -533,6 +897,9 @@ define void @constant_sextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <1 } ; FUNC-LABEL: {{^}}constant_zextload_v32i8_to_v32i16: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @constant_zextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = zext <32 x i8> %load to <32 x i16> @@ -541,6 +908,41 @@ define void @constant_zextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <3 } ; FUNC-LABEL: {{^}}constant_sextload_v32i8_to_v32i16: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @constant_sextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <32 x i8> addrspace(2)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(2)* %in %ext = sext <32 x i8> %load to <32 x i16> diff --git a/test/CodeGen/AMDGPU/load-global-i16.ll b/test/CodeGen/AMDGPU/load-global-i16.ll index 11e6b10c38ff..f398dd32e06d 100644 --- a/test/CodeGen/AMDGPU/load-global-i16.ll +++ b/test/CodeGen/AMDGPU/load-global-i16.ll @@ -1,6 +1,6 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,GCN-NOHSA-SI,FUNC %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-HSA,FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,GCN-NOHSA-VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s @@ -10,7 +10,7 @@ ; GCN-NOHSA: buffer_load_ushort v{{[0-9]+}} ; GCN-HSA: flat_load_ushort -; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0 +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_load_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) { entry: %ld = load i16, i16 addrspace(1)* %in @@ -22,7 +22,7 @@ entry: ; GCN-NOHSA: buffer_load_dword v ; GCN-HSA: flat_load_dword v -; EG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_load_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) { entry: %ld = load <2 x i16>, <2 x i16> addrspace(1)* %in @@ -34,8 +34,8 @@ entry: ; GCN-NOHSA: buffer_load_dwordx2 v ; GCN-HSA: flat_load_dwordx2 v -; EG-DAG: VTX_READ_32 -; EG-DAG: VTX_READ_16 +; EG-DAG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 4, #1 define void @global_load_v3i16(<3 x i16> addrspace(1)* %out, <3 x i16> addrspace(1)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(1)* %in @@ -47,7 +47,7 @@ entry: ; GCN-NOHSA: buffer_load_dwordx2 ; GCN-HSA: flat_load_dwordx2 -; EG: VTX_READ_64 +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_load_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) { entry: %ld = load <4 x i16>, <4 x i16> addrspace(1)* %in @@ -59,7 +59,7 @@ entry: ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 -; EG: VTX_READ_128 +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_load_v8i16(<8 x i16> addrspace(1)* %out, <8 x i16> addrspace(1)* %in) { entry: %ld = load <8 x i16>, <8 x i16> addrspace(1)* %in @@ -74,8 +74,8 @@ entry: ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 -; EG: VTX_READ_128 -; EG: VTX_READ_128 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_load_v16i16(<16 x i16> addrspace(1)* %out, <16 x i16> addrspace(1)* %in) { entry: %ld = load <16 x i16>, <16 x i16> addrspace(1)* %in @@ -90,7 +90,7 @@ entry: ; GCN-HSA: flat_load_ushort ; GCN-HSA: flat_store_dword -; EG: VTX_READ_16 T{{[0-9]+\.X, T[0-9]+\.X}} +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in) #0 { %a = load i16, i16 addrspace(1)* %in %ext = zext i16 %a to i32 @@ -105,7 +105,7 @@ define void @global_zextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1) ; GCN-HSA: flat_load_sshort ; GCN-HSA: flat_store_dword -; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], [[DST]] +; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], T{{[0-9]+}}.X, 0, #1 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal ; EG: 16 define void @global_sextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in) #0 { @@ -118,6 +118,8 @@ define void @global_sextload_i16_to_i32(i32 addrspace(1)* %out, i16 addrspace(1) ; FUNC-LABEL: {{^}}global_zextload_v1i16_to_v1i32: ; GCN-NOHSA: buffer_load_ushort ; GCN-HSA: flat_load_ushort + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(1)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(1)* %in %ext = zext <1 x i16> %load to <1 x i32> @@ -128,6 +130,10 @@ define void @global_zextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i ; FUNC-LABEL: {{^}}global_sextload_v1i16_to_v1i32: ; GCN-NOHSA: buffer_load_sshort ; GCN-HSA: flat_load_sshort + +; EG: VTX_READ_16 [[DST:T[0-9]\.[XYZW]]], T{{[0-9]+}}.X, 0, #1 +; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal +; EG: 16 define void @global_sextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i16> addrspace(1)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(1)* %in %ext = sext <1 x i16> %load to <1 x i32> @@ -138,6 +144,11 @@ define void @global_sextload_v1i16_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i ; FUNC-LABEL: {{^}}global_zextload_v2i16_to_v2i32: ; GCN-NOHSA: buffer_load_dword ; GCN-HSA: flat_load_dword + +; EG: VTX_READ_32 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; TODO: This should use DST, but for some there are redundant MOVs +; EG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, literal +; EG: 16 define void @global_zextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(1)* %in %ext = zext <2 x i16> %load to <2 x i32> @@ -150,10 +161,11 @@ define void @global_zextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i ; GCN-HSA: flat_load_dword -; EG-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal +; EG: VTX_READ_32 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; TODO: We should also use ASHR instead of LSHR + BFE +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, 0.0, literal ; EG-DAG: 16 ; EG-DAG: 16 define void @global_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 { @@ -166,6 +178,12 @@ define void @global_sextload_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i ; FUNC-LABEL: {{^}}global_global_zextload_v3i16_to_v3i32: ; GCN-NOHSA: buffer_load_dwordx2 ; GCN-HSA: flat_load_dwordx2 + +; EG-DAG: VTX_READ_32 [[DST_HI:T[0-9]\.[XYZW]]], [[DST_HI]], 0, #1 +; EG-DAG: VTX_READ_16 [[DST_LO:T[0-9]\.[XYZW]]], [[DST_LO]], 4, #1 +; TODO: This should use DST, but for some there are redundant MOVs +; EG: LSHR {{[* ]*}}{{T[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG: 16 define void @global_global_zextload_v3i16_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i16> addrspace(1)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(1)* %in @@ -177,6 +195,15 @@ entry: ; FUNC-LABEL: {{^}}global_global_sextload_v3i16_to_v3i32: ; GCN-NOHSA: buffer_load_dwordx2 ; GCN-HSA: flat_load_dwordx2 + +; EG-DAG: VTX_READ_32 [[DST_HI:T[0-9]\.[XYZW]]], [[DST_HI]], 0, #1 +; EG-DAG: VTX_READ_16 [[DST_LO:T[0-9]\.[XYZW]]], [[DST_LO]], 4, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: ASHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{PV.[XYZW]}}, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, 0.0, literal +; EG-DAG: 16 +; EG-DAG: 16 define void @global_global_sextload_v3i16_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i16> addrspace(1)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(1)* %in @@ -190,10 +217,13 @@ entry: ; GCN-HSA: flat_load_dwordx2 -; EG: VTX_READ_16 -; EG: VTX_READ_16 -; EG: VTX_READ_16 -; EG: VTX_READ_16 +; EG: VTX_READ_64 [[DST:T[0-9]\.XY]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG-DAG: 16 +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG-DAG: AND_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{T[0-9].[XYZW]}}, literal +; EG-DAG: 16 define void @global_global_zextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(1)* %in %ext = zext <4 x i16> %load to <4 x i32> @@ -206,14 +236,13 @@ define void @global_global_zextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, ; GCN-HSA: flat_load_dwordx2 -; EG-DAG: VTX_READ_16 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_16 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: VTX_READ_16 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]] -; EG-DAG: VTX_READ_16 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal +; EG: VTX_READ_64 [[DST:T[0-9]\.XY]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; TODO: We should use ASHR instead of LSHR + BFE +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 16 ; EG-DAG: 16 ; EG-DAG: 16 @@ -228,6 +257,17 @@ define void @global_sextload_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i ; FUNC-LABEL: {{^}}global_zextload_v8i16_to_v8i32: ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG: VTX_READ_128 [[DST:T[0-9]\.XYZW]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 define void @global_zextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(1)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(1)* %in %ext = zext <8 x i16> %load to <8 x i32> @@ -238,6 +278,25 @@ define void @global_zextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i ; FUNC-LABEL: {{^}}global_sextload_v8i16_to_v8i32: ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG: VTX_READ_128 [[DST:T[0-9]\.XYZW]], {{T[0-9].[XYZW]}}, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: LSHR {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 +; EG-DAG: 16 define void @global_sextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i16> addrspace(1)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(1)* %in %ext = sext <8 x i16> %load to <8 x i32> @@ -251,6 +310,9 @@ define void @global_sextload_v8i16_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 define void @global_zextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(1)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(1)* %in %ext = zext <16 x i16> %load to <16 x i32> @@ -259,6 +321,9 @@ define void @global_zextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_sextload_v16i16_to_v16i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 define void @global_sextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i16> addrspace(1)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(1)* %in %ext = sext <16 x i16> %load to <16 x i32> @@ -276,6 +341,11 @@ define void @global_sextload_v16i16_to_v16i32(<16 x i32> addrspace(1)* %out, <16 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 48, #1 define void @global_zextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(1)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(1)* %in %ext = zext <32 x i16> %load to <32 x i32> @@ -293,6 +363,11 @@ define void @global_zextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 48, #1 define void @global_sextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i16> addrspace(1)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(1)* %in %ext = sext <32 x i16> %load to <32 x i32> @@ -318,6 +393,15 @@ define void @global_sextload_v32i16_to_v32i32(<32 x i32> addrspace(1)* %out, <32 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 48, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 64, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 80, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 96, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 112, #1 define void @global_zextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(1)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(1)* %in %ext = zext <64 x i16> %load to <64 x i32> @@ -326,6 +410,15 @@ define void @global_zextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 } ; FUNC-LABEL: {{^}}global_sextload_v64i16_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 48, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 64, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 80, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 96, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, {{T[0-9]+.[XYZW]}}, 112, #1 define void @global_sextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i16> addrspace(1)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(1)* %in %ext = sext <64 x i16> %load to <64 x i32> @@ -340,6 +433,9 @@ define void @global_sextload_v64i16_to_v64i32(<64 x i32> addrspace(1)* %out, <64 ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @global_zextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in) #0 { %a = load i16, i16 addrspace(1)* %in %ext = zext i16 %a to i64 @@ -348,12 +444,24 @@ define void @global_zextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1) } ; FUNC-LABEL: {{^}}global_sextload_i16_to_i64: -; GCN-NOHSA-DAG: buffer_load_sshort v[[LO:[0-9]+]], +; FIXME: Need to optimize this sequence to avoid extra bfe: +; t28: i32,ch = load<LD2[%in(addrspace=1)], anyext from i16> t12, t27, undef:i64 +; t31: i64 = any_extend t28 +; t33: i64 = sign_extend_inreg t31, ValueType:ch:i16 + +; GCN-NOHSA-SI-DAG: buffer_load_sshort v[[LO:[0-9]+]], ; GCN-HSA-DAG: flat_load_sshort v[[LO:[0-9]+]], +; GCN-NOHSA-VI-DAG: buffer_load_ushort v[[ULO:[0-9]+]], +; GCN-NOHSA-VI-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16 ; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]] ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 15 ? +; EG: 31 define void @global_sextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in) #0 { %a = load i16, i16 addrspace(1)* %in %ext = sext i16 %a to i64 @@ -362,6 +470,9 @@ define void @global_sextload_i16_to_i64(i64 addrspace(1)* %out, i16 addrspace(1) } ; FUNC-LABEL: {{^}}global_zextload_v1i16_to_v1i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @global_zextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(1)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(1)* %in %ext = zext <1 x i16> %load to <1 x i64> @@ -370,6 +481,11 @@ define void @global_zextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i } ; FUNC-LABEL: {{^}}global_sextload_v1i16_to_v1i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 15 ? +; EG: 31 define void @global_sextload_v1i16_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i16> addrspace(1)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(1)* %in %ext = sext <1 x i16> %load to <1 x i64> @@ -386,6 +502,8 @@ define void @global_zextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i } ; FUNC-LABEL: {{^}}global_sextload_v2i16_to_v2i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(1)* %in %ext = sext <2 x i16> %load to <2 x i64> @@ -394,6 +512,8 @@ define void @global_sextload_v2i16_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i } ; FUNC-LABEL: {{^}}global_zextload_v4i16_to_v4i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(1)* %in %ext = zext <4 x i16> %load to <4 x i64> @@ -402,6 +522,8 @@ define void @global_zextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i } ; FUNC-LABEL: {{^}}global_sextload_v4i16_to_v4i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(1)* %in %ext = sext <4 x i16> %load to <4 x i64> @@ -410,6 +532,8 @@ define void @global_sextload_v4i16_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i } ; FUNC-LABEL: {{^}}global_zextload_v8i16_to_v8i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(1)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(1)* %in %ext = zext <8 x i16> %load to <8 x i64> @@ -418,6 +542,8 @@ define void @global_zextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i } ; FUNC-LABEL: {{^}}global_sextload_v8i16_to_v8i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i16> addrspace(1)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(1)* %in %ext = sext <8 x i16> %load to <8 x i64> @@ -426,6 +552,9 @@ define void @global_sextload_v8i16_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i } ; FUNC-LABEL: {{^}}global_zextload_v16i16_to_v16i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_zextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(1)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(1)* %in %ext = zext <16 x i16> %load to <16 x i64> @@ -434,6 +563,9 @@ define void @global_zextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_sextload_v16i16_to_v16i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_sextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i16> addrspace(1)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(1)* %in %ext = sext <16 x i16> %load to <16 x i64> @@ -442,6 +574,11 @@ define void @global_sextload_v16i16_to_v16i64(<16 x i64> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_zextload_v32i16_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 48, #1 define void @global_zextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(1)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(1)* %in %ext = zext <32 x i16> %load to <32 x i64> @@ -450,6 +587,11 @@ define void @global_zextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 } ; FUNC-LABEL: {{^}}global_sextload_v32i16_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 48, #1 define void @global_sextload_v32i16_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i16> addrspace(1)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(1)* %in %ext = sext <32 x i16> %load to <32 x i64> diff --git a/test/CodeGen/AMDGPU/load-global-i32.ll b/test/CodeGen/AMDGPU/load-global-i32.ll index 5e1171a69be5..86f5d4eb4d5d 100644 --- a/test/CodeGen/AMDGPU/load-global-i32.ll +++ b/test/CodeGen/AMDGPU/load-global-i32.ll @@ -360,7 +360,7 @@ define void @global_zextload_v16i32_to_v16i64(<16 x i64> addrspace(1)* %out, <16 ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 ; GCN-NOHSA: buffer_load_dwordx4 -; GCN-NOHSA: buffer_load_dwordx4 +; GCN-NOHSA-DAG: buffer_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 ; GCN-HSA: flat_load_dwordx4 diff --git a/test/CodeGen/AMDGPU/load-global-i8.ll b/test/CodeGen/AMDGPU/load-global-i8.ll index b697967f1a23..b183b6ccd622 100644 --- a/test/CodeGen/AMDGPU/load-global-i8.ll +++ b/test/CodeGen/AMDGPU/load-global-i8.ll @@ -1,6 +1,6 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s -; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-HSA -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-NOHSA -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,SI,FUNC %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-HSA,SI,FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GCN-NOHSA,VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s @@ -9,7 +9,8 @@ ; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}} ; GCN-HSA: flat_load_ubyte -; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0 +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: NOT AND define void @global_load_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { entry: %ld = load i8, i8 addrspace(1)* %in @@ -21,7 +22,7 @@ entry: ; GCN-NOHSA: buffer_load_ushort v ; GCN-HSA: flat_load_ushort v -; EG: VTX_READ_16 +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_load_v2i8(<2 x i8> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { entry: %ld = load <2 x i8>, <2 x i8> addrspace(1)* %in @@ -33,7 +34,7 @@ entry: ; GCN-NOHSA: buffer_load_dword v ; GCN-HSA: flat_load_dword v -; EG-DAG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_load_v3i8(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(1)* %in @@ -45,7 +46,7 @@ entry: ; GCN-NOHSA: buffer_load_dword v ; GCN-HSA: flat_load_dword v -; EG: VTX_READ_32 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_load_v4i8(<4 x i8> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { entry: %ld = load <4 x i8>, <4 x i8> addrspace(1)* %in @@ -57,7 +58,7 @@ entry: ; GCN-NOHSA: buffer_load_dwordx2 ; GCN-HSA: flat_load_dwordx2 -; EG: VTX_READ_64 +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_load_v8i8(<8 x i8> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { entry: %ld = load <8 x i8>, <8 x i8> addrspace(1)* %in @@ -70,7 +71,7 @@ entry: ; GCN-HSA: flat_load_dwordx4 -; EG: VTX_READ_128 +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_load_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { entry: %ld = load <16 x i8>, <16 x i8> addrspace(1)* %in @@ -82,7 +83,7 @@ entry: ; GCN-NOHSA: buffer_load_ubyte v{{[0-9]+}}, ; GCN-HSA: flat_load_ubyte -; EG: VTX_READ_8 T{{[0-9]+\.X, T[0-9]+\.X}} +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { %a = load i8, i8 addrspace(1)* %in %ext = zext i8 %a to i32 @@ -94,7 +95,7 @@ define void @global_zextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(1)* ; GCN-NOHSA: buffer_load_sbyte ; GCN-HSA: flat_load_sbyte -; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]] +; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 ; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal ; EG: 8 define void @global_sextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { @@ -105,6 +106,8 @@ define void @global_sextload_i8_to_i32(i32 addrspace(1)* %out, i8 addrspace(1)* } ; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i32: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = zext <1 x i8> %load to <1 x i32> @@ -113,6 +116,10 @@ define void @global_zextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i32: + +; EG: VTX_READ_8 [[DST:T[0-9]\.[XYZW]]], [[DST]], 0, #1 +; EG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST]], 0.0, literal +; EG: 8 define void @global_sextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = sext <1 x i8> %load to <1 x i32> @@ -124,8 +131,10 @@ define void @global_sextload_v1i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i8 ; GCN-NOHSA: buffer_load_ushort ; GCN-HSA: flat_load_ushort -; EG: VTX_READ_8 -; EG: VTX_READ_8 +; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, literal +; EG-DAG: 8 define void @global_zextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in %ext = zext <2 x i8> %load to <2 x i32> @@ -137,10 +146,10 @@ define void @global_zextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8 ; GCN-NOHSA: buffer_load_ushort ; GCN-HSA: flat_load_ushort -; EG-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal +; EG: VTX_READ_16 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 8 ; EG-DAG: 8 define void @global_sextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { @@ -154,9 +163,17 @@ define void @global_sextload_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i8 ; GCN-NOHSA: buffer_load_dword v ; GCN-HSA: flat_load_dword v -; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; SI-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; VI-DAG: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, v{{[0-9]+}} ; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 8 ; GCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff, + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 define void @global_zextload_v3i8_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(1)* %in @@ -169,9 +186,27 @@ entry: ; GCN-NOHSA: buffer_load_dword v ; GCN-HSA: flat_load_dword v -; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +;FIXME: Need to optimize this sequence to avoid extra shift on VI. + +; t23: i16 = truncate t18 +; t49: i16 = srl t23, Constant:i32<8> +; t57: i32 = any_extend t49 +; t58: i32 = sign_extend_inreg t57, ValueType:ch:i8 + +; SI-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; VI-DAG: v_lshrrev_b16_e32 [[SHIFT:v[0-9]+]], 8, v{{[0-9]+}} +; VI-DAG: v_bfe_i32 v{{[0-9]+}}, [[SHIFT]], 0, 8 ; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 ; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 8 + +; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_sextload_v3i8_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i8> addrspace(1)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(1)* %in @@ -184,10 +219,14 @@ entry: ; GCN-NOHSA: buffer_load_dword ; GCN-HSA: flat_load_dword -; EG: VTX_READ_8 -; EG: VTX_READ_8 -; EG: VTX_READ_8 -; EG: VTX_READ_8 +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_zextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in %ext = zext <4 x i8> %load to <4 x i32> @@ -199,14 +238,12 @@ define void @global_zextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i8 ; GCN-NOHSA: buffer_load_dword ; GCN-HSA: flat_load_dword -; EG-DAG: VTX_READ_8 [[DST_X:T[0-9]\.[XYZW]]], [[DST_X]] -; EG-DAG: VTX_READ_8 [[DST_Y:T[0-9]\.[XYZW]]], [[DST_Y]] -; EG-DAG: VTX_READ_8 [[DST_Z:T[0-9]\.[XYZW]]], [[DST_Z]] -; EG-DAG: VTX_READ_8 [[DST_W:T[0-9]\.[XYZW]]], [[DST_W]] -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_X]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Y]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_Z]], 0.0, literal -; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, [[DST_W]], 0.0, literal +; EG: VTX_READ_32 [[DST:T[0-9]+\.X]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal ; EG-DAG: 8 ; EG-DAG: 8 ; EG-DAG: 8 @@ -219,6 +256,23 @@ define void @global_sextload_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i32: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_zextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = zext <8 x i8> %load to <8 x i32> @@ -227,6 +281,25 @@ define void @global_zextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i32: + +; EG: VTX_READ_64 [[DST:T[0-9]+\.XY]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_sextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = sext <8 x i8> %load to <8 x i32> @@ -235,6 +308,39 @@ define void @global_sextload_v8i8_to_v8i32(<8 x i32> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i32: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_zextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = zext <16 x i8> %load to <16 x i32> @@ -243,6 +349,41 @@ define void @global_zextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i32: + +; EG: VTX_READ_128 [[DST:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_sextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = sext <16 x i8> %load to <16 x i32> @@ -251,6 +392,70 @@ define void @global_sextload_v16i8_to_v16i32(<16 x i32> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i32: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: BFE_UINT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, {{.*}}literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_zextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = zext <32 x i8> %load to <32 x i32> @@ -259,6 +464,74 @@ define void @global_zextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 } ; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i32: + +; EG-DAG: VTX_READ_128 [[DST_LO:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 [[DST_HI:T[0-9]+\.XYZW]], T{{[0-9]+}}.X, 16, #1 +; TODO: These should use DST, but for some there are redundant MOVs +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9]+.[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 +; EG-DAG: 8 define void @global_sextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = sext <32 x i8> %load to <32 x i32> @@ -267,6 +540,11 @@ define void @global_sextload_v32i8_to_v32i32(<32 x i32> addrspace(1)* %out, <32 } ; FUNC-LABEL: {{^}}global_zextload_v64i8_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1 define void @global_zextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i8> addrspace(1)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(1)* %in %ext = zext <64 x i8> %load to <64 x i32> @@ -275,6 +553,11 @@ define void @global_zextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 } ; FUNC-LABEL: {{^}}global_sextload_v64i8_to_v64i32: + +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 32, #1 +; EG-DAG: VTX_READ_128 {{T[0-9]+\.XYZW}}, T{{[0-9]+}}.X, 48, #1 define void @global_sextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 x i8> addrspace(1)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(1)* %in %ext = sext <64 x i8> %load to <64 x i32> @@ -290,6 +573,9 @@ define void @global_sextload_v64i8_to_v64i32(<64 x i32> addrspace(1)* %out, <64 ; GCN-HSA-DAG: flat_load_ubyte v[[LO:[0-9]+]], ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]] + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @global_zextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { %a = load i8, i8 addrspace(1)* %in %ext = zext i8 %a to i64 @@ -304,6 +590,11 @@ define void @global_zextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(1)* ; GCN-NOHSA: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} ; GCN-HSA: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 7 ? +; EG: 31 define void @global_sextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { %a = load i8, i8 addrspace(1)* %in %ext = sext i8 %a to i64 @@ -312,6 +603,9 @@ define void @global_sextload_i8_to_i64(i64 addrspace(1)* %out, i8 addrspace(1)* } ; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i64: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: MOV {{.*}}, 0.0 define void @global_zextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = zext <1 x i8> %load to <1 x i64> @@ -320,6 +614,11 @@ define void @global_zextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i64: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG: ASHR {{\**}} {{T[0-9]\.[XYZW]}}, {{.*}}, literal +; TODO: Why not 7 ? +; EG: 31 define void @global_sextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = sext <1 x i8> %load to <1 x i64> @@ -328,6 +627,8 @@ define void @global_sextload_v1i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v2i8_to_v2i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in %ext = zext <2 x i8> %load to <2 x i64> @@ -336,6 +637,8 @@ define void @global_zextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v2i8_to_v2i64: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in %ext = sext <2 x i8> %load to <2 x i64> @@ -344,6 +647,8 @@ define void @global_sextload_v2i8_to_v2i64(<2 x i64> addrspace(1)* %out, <2 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v4i8_to_v4i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in %ext = zext <4 x i8> %load to <4 x i64> @@ -352,6 +657,8 @@ define void @global_zextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v4i8_to_v4i64: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in %ext = sext <4 x i8> %load to <4 x i64> @@ -360,6 +667,8 @@ define void @global_sextload_v4i8_to_v4i64(<4 x i64> addrspace(1)* %out, <4 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = zext <8 x i8> %load to <8 x i64> @@ -368,6 +677,8 @@ define void @global_zextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i64: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = sext <8 x i8> %load to <8 x i64> @@ -376,6 +687,8 @@ define void @global_sextload_v8i8_to_v8i64(<8 x i64> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = zext <16 x i8> %load to <16 x i64> @@ -384,6 +697,8 @@ define void @global_zextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i64: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_sextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = sext <16 x i8> %load to <16 x i64> @@ -392,6 +707,9 @@ define void @global_sextload_v16i8_to_v16i64(<16 x i64> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_zextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = zext <32 x i8> %load to <32 x i64> @@ -400,6 +718,9 @@ define void @global_zextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 } ; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i64: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_sextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = sext <32 x i8> %load to <32 x i64> @@ -429,6 +750,8 @@ define void @global_sextload_v32i8_to_v32i64(<32 x i64> addrspace(1)* %out, <32 ; GCN-HSA: flat_load_ubyte v[[VAL:[0-9]+]], ; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]] + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { %a = load i8, i8 addrspace(1)* %in %ext = zext i8 %a to i16 @@ -442,6 +765,9 @@ define void @global_zextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(1)* ; GCN-NOHSA: buffer_store_short v[[VAL]] ; GCN-HSA: flat_store_short v{{\[[0-9]+:[0-9]+\]}}, v[[VAL]] + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(1)* %in) #0 { %a = load i8, i8 addrspace(1)* %in %ext = sext i8 %a to i16 @@ -450,6 +776,8 @@ define void @global_sextload_i8_to_i16(i16 addrspace(1)* %out, i8 addrspace(1)* } ; FUNC-LABEL: {{^}}global_zextload_v1i8_to_v1i16: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = zext <1 x i8> %load to <1 x i16> @@ -458,6 +786,9 @@ define void @global_zextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v1i8_to_v1i16: + +; EG: VTX_READ_8 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x i8> addrspace(1)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(1)* %in %ext = sext <1 x i8> %load to <1 x i16> @@ -466,6 +797,8 @@ define void @global_sextload_v1i8_to_v1i16(<1 x i16> addrspace(1)* %out, <1 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v2i8_to_v2i16: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in %ext = zext <2 x i8> %load to <2 x i16> @@ -474,6 +807,10 @@ define void @global_zextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v2i8_to_v2i16: + +; EG: VTX_READ_16 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8> addrspace(1)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(1)* %in %ext = sext <2 x i8> %load to <2 x i16> @@ -482,6 +819,8 @@ define void @global_sextload_v2i8_to_v2i16(<2 x i16> addrspace(1)* %out, <2 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v4i8_to_v4i16: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in %ext = zext <4 x i8> %load to <4 x i16> @@ -490,6 +829,12 @@ define void @global_zextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v4i8_to_v4i16: + +; EG: VTX_READ_32 T{{[0-9]+}}.X, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8> addrspace(1)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(1)* %in %ext = sext <4 x i8> %load to <4 x i16> @@ -498,6 +843,8 @@ define void @global_sextload_v4i8_to_v4i16(<4 x i16> addrspace(1)* %out, <4 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v8i8_to_v8i16: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = zext <8 x i8> %load to <8 x i16> @@ -506,6 +853,16 @@ define void @global_zextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_sextload_v8i8_to_v8i16: + +; EG: VTX_READ_64 T{{[0-9]+}}.XY, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8> addrspace(1)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(1)* %in %ext = sext <8 x i8> %load to <8 x i16> @@ -514,6 +871,8 @@ define void @global_sextload_v8i8_to_v8i16(<8 x i16> addrspace(1)* %out, <8 x i8 } ; FUNC-LABEL: {{^}}global_zextload_v16i8_to_v16i16: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 define void @global_zextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = zext <16 x i8> %load to <16 x i16> @@ -522,6 +881,24 @@ define void @global_zextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_sextload_v16i8_to_v16i16: + +; EG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 x i8> addrspace(1)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(1)* %in %ext = sext <16 x i8> %load to <16 x i16> @@ -530,6 +907,9 @@ define void @global_sextload_v16i8_to_v16i16(<16 x i16> addrspace(1)* %out, <16 } ; FUNC-LABEL: {{^}}global_zextload_v32i8_to_v32i16: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 define void @global_zextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = zext <32 x i8> %load to <32 x i16> @@ -538,6 +918,41 @@ define void @global_zextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <32 } ; FUNC-LABEL: {{^}}global_sextload_v32i8_to_v32i16: + +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 0, #1 +; EG-DAG: VTX_READ_128 T{{[0-9]+}}.XYZW, T{{[0-9]+}}.X, 16, #1 +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal +; EG-DAG: BFE_INT {{[* ]*}}T{{[0-9].[XYZW]}}, {{.*}}, 0.0, literal define void @global_sextload_v32i8_to_v32i16(<32 x i16> addrspace(1)* %out, <32 x i8> addrspace(1)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(1)* %in %ext = sext <32 x i8> %load to <32 x i16> diff --git a/test/CodeGen/AMDGPU/load-local-i16.ll b/test/CodeGen/AMDGPU/load-local-i16.ll index d3c0af469dd2..d4e86de66aff 100644 --- a/test/CodeGen/AMDGPU/load-local-i16.ll +++ b/test/CodeGen/AMDGPU/load-local-i16.ll @@ -1,11 +1,15 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s ; FUNC-LABEL: {{^}}local_load_i16: ; GCN: ds_read_u16 v{{[0-9]+}} -; EG: LDS_USHORT_READ_RET +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG: LDS_SHORT_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_load_i16(i16 addrspace(3)* %out, i16 addrspace(3)* %in) { entry: %ld = load i16, i16 addrspace(3)* %in @@ -16,7 +20,11 @@ entry: ; FUNC-LABEL: {{^}}local_load_v2i16: ; GCN: ds_read_b32 -; EG: LDS_READ_RET +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_load_v2i16(<2 x i16> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) { entry: %ld = load <2 x i16>, <2 x i16> addrspace(3)* %in @@ -51,7 +59,7 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v8i16: -; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1{{$}} +; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} ; EG: LDS_READ_RET ; EG: LDS_READ_RET @@ -65,8 +73,8 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v16i16: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:3{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}} ; EG: LDS_READ_RET @@ -89,7 +97,11 @@ entry: ; GCN: ds_read_u16 ; GCN: ds_write_b32 -; EG: LDS_USHORT_READ_RET +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_zextload_i16_to_i32(i32 addrspace(3)* %out, i16 addrspace(3)* %in) #0 { %a = load i16, i16 addrspace(3)* %in %ext = zext i16 %a to i32 @@ -102,8 +114,13 @@ define void @local_zextload_i16_to_i32(i32 addrspace(3)* %out, i16 addrspace(3)* ; GCN: s_mov_b32 m0 ; GCN: ds_read_i16 -; EG: LDS_USHORT_READ_RET -; EG: BFE_INT +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[TMP:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: BFE_INT {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], {{.*}}, 0.0, literal +; EG: 16 +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_sextload_i16_to_i32(i32 addrspace(3)* %out, i16 addrspace(3)* %in) #0 { %a = load i16, i16 addrspace(3)* %in %ext = sext i16 %a to i32 @@ -113,6 +130,12 @@ define void @local_sextload_i16_to_i32(i32 addrspace(3)* %out, i16 addrspace(3)* ; FUNC-LABEL: {{^}}local_zextload_v1i16_to_v1i32: ; GCN: ds_read_u16 + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_zextload_v1i16_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i16> addrspace(3)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(3)* %in %ext = zext <1 x i16> %load to <1 x i32> @@ -122,6 +145,14 @@ define void @local_zextload_v1i16_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i1 ; FUNC-LABEL: {{^}}local_sextload_v1i16_to_v1i32: ; GCN: ds_read_i16 + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[TMP:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: BFE_INT {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], {{.*}}, 0.0, literal +; EG: 16 +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_sextload_v1i16_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i16> addrspace(3)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(3)* %in %ext = sext <1 x i16> %load to <1 x i32> @@ -134,8 +165,7 @@ define void @local_sextload_v1i16_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i1 ; GCN: s_mov_b32 m0 ; GCN: ds_read_b32 -; EG: LDS_USHORT_READ_RET -; EG: LDS_USHORT_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v2i16_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(3)* %in %ext = zext <2 x i16> %load to <2 x i32> @@ -148,10 +178,9 @@ define void @local_zextload_v2i16_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i1 ; GCN: s_mov_b32 m0 ; GCN: ds_read_b32 -; EG-DAG: LDS_USHORT_READ_RET -; EG-DAG: LDS_USHORT_READ_RET -; EG-DAG: BFE_INT -; EG-DAG: BFE_INT +; EG: LDS_READ_RET +; EG: BFE_INT +; EG: BFE_INT define void @local_sextload_v2i16_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(3)* %in %ext = sext <2 x i16> %load to <2 x i32> @@ -163,6 +192,8 @@ define void @local_sextload_v2i16_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i1 ; GCN: ds_read_b64 ; GCN-DAG: ds_write_b32 ; GCN-DAG: ds_write_b64 + +; EG: LDS_READ_RET define void @local_local_zextload_v3i16_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i16> addrspace(3)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(3)* %in @@ -175,6 +206,11 @@ entry: ; GCN: ds_read_b64 ; GCN-DAG: ds_write_b32 ; GCN-DAG: ds_write_b64 + +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_local_sextload_v3i16_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i16> addrspace(3)* %in) { entry: %ld = load <3 x i16>, <3 x i16> addrspace(3)* %in @@ -188,10 +224,8 @@ entry: ; GCN: s_mov_b32 m0 ; GCN: ds_read_b64 -; EG: LDS_USHORT_READ_RET -; EG: LDS_USHORT_READ_RET -; EG: LDS_USHORT_READ_RET -; EG: LDS_USHORT_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_local_zextload_v4i16_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i16> addrspace(3)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(3)* %in %ext = zext <4 x i16> %load to <4 x i32> @@ -204,10 +238,8 @@ define void @local_local_zextload_v4i16_to_v4i32(<4 x i32> addrspace(3)* %out, < ; GCN: s_mov_b32 m0 ; GCN: ds_read_b64 -; EG-DAG: LDS_USHORT_READ_RET -; EG-DAG: LDS_USHORT_READ_RET -; EG-DAG: LDS_USHORT_READ_RET -; EG-DAG: LDS_USHORT_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET ; EG-DAG: BFE_INT ; EG-DAG: BFE_INT ; EG-DAG: BFE_INT @@ -221,6 +253,11 @@ define void @local_sextload_v4i16_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i1 ; FUNC-LABEL: {{^}}local_zextload_v8i16_to_v8i32: ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(3)* %in %ext = zext <8 x i16> %load to <8 x i32> @@ -230,6 +267,19 @@ define void @local_zextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i1 ; FUNC-LABEL: {{^}}local_sextload_v8i16_to_v8i32: ; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(3)* %in %ext = sext <8 x i16> %load to <8 x i32> @@ -237,16 +287,23 @@ define void @local_sextload_v8i16_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i1 ret void } -; FIXME: Should have 2 ds_read_b64 ; FUNC-LABEL: {{^}}local_zextload_v16i16_to_v16i32: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}} -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+$}} -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:24 +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}} ; GCN: ds_write2_b64 ; GCN: ds_write2_b64 ; GCN: ds_write2_b64 ; GCN: ds_write2_b64 + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(3)* %in %ext = zext <16 x i16> %load to <16 x i32> @@ -255,9 +312,34 @@ define void @local_zextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 } ; FUNC-LABEL: {{^}}local_sextload_v16i16_to_v16i32: -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:3 offset1:1{{$}} -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16{{$}} + +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}} + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(3)* %in %ext = sext <16 x i16> %load to <16 x i32> @@ -270,6 +352,23 @@ define void @local_sextload_v16i16_to_v16i32(<16 x i32> addrspace(3)* %out, <16 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7 + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(3)* %in %ext = zext <32 x i16> %load to <32 x i32> @@ -278,10 +377,35 @@ define void @local_zextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 } ; FUNC-LABEL: {{^}}local_sextload_v32i16_to_v32i32: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:3 offset1:4 -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:5{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5 +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:12 offset1:13 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:10 offset1:11 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:8 offset1:9 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:6 offset1:7 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:4 offset1:5 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset1:1 + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_sextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(3)* %in %ext = sext <32 x i16> %load to <32 x i32> @@ -289,17 +413,64 @@ define void @local_sextload_v32i16_to_v32i32(<32 x i32> addrspace(3)* %out, <32 ret void } -; FIXME: Missed read2 ; FUNC-LABEL: {{^}}local_zextload_v64i16_to_v64i32: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:11 offset1:15 +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:14 offset1:15 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7 -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:64 -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:9 offset1:10 +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:8 offset1:9 ; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:12 offset1:13 -; GCN-DAG: ds_read_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:112 +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:10 offset1:11 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:26 offset1:27 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:24 offset1:25 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:22 offset1:23 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:20 offset1:21 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:18 offset1:19 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:16 offset1:17 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:12 offset1:13 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:10 offset1:11 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:8 offset1:9 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:6 offset1:7 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:4 offset1:5 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset1:1 + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v64i16_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i16> addrspace(3)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(3)* %in %ext = zext <64 x i16> %load to <64 x i32> @@ -308,6 +479,39 @@ define void @local_zextload_v64i16_to_v64i32(<64 x i32> addrspace(3)* %out, <64 } ; FUNC-LABEL: {{^}}local_sextload_v64i16_to_v64i32: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_sextload_v64i16_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i16> addrspace(3)* %in) #0 { %load = load <64 x i16>, <64 x i16> addrspace(3)* %in %ext = sext <64 x i16> %load to <64 x i32> @@ -320,6 +524,13 @@ define void @local_sextload_v64i16_to_v64i32(<64 x i32> addrspace(3)* %out, <64 ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}} ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]] + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: LDS_WRITE +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_zextload_i16_to_i64(i64 addrspace(3)* %out, i16 addrspace(3)* %in) #0 { %a = load i16, i16 addrspace(3)* %in %ext = zext i16 %a to i64 @@ -328,10 +539,25 @@ define void @local_zextload_i16_to_i64(i64 addrspace(3)* %out, i16 addrspace(3)* } ; FUNC-LABEL: {{^}}local_sextload_i16_to_i64: -; GCN: ds_read_i16 v[[LO:[0-9]+]], +; FIXME: Need to optimize this sequence to avoid an extra shift. +; t25: i32,ch = load<LD2[%in(addrspace=3)], anyext from i16> t12, t10, undef:i32 +; t28: i64 = any_extend t25 +; t30: i64 = sign_extend_inreg t28, ValueType:ch:i16 +; SI: ds_read_i16 v[[LO:[0-9]+]], +; VI: ds_read_u16 v[[ULO:[0-9]+]] +; VI: v_bfe_i32 v[[LO:[0-9]+]], v[[ULO]], 0, 16 ; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]] + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[TMP:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: BFE_INT {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], {{.*}}, 0.0, literal +; EG-DAG: LDS_WRITE +; EG-DAG: 16 +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_sextload_i16_to_i64(i64 addrspace(3)* %out, i16 addrspace(3)* %in) #0 { %a = load i16, i16 addrspace(3)* %in %ext = sext i16 %a to i64 @@ -340,6 +566,13 @@ define void @local_sextload_i16_to_i64(i64 addrspace(3)* %out, i16 addrspace(3)* } ; FUNC-LABEL: {{^}}local_zextload_v1i16_to_v1i64: + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: LDS_WRITE +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_zextload_v1i16_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i16> addrspace(3)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(3)* %in %ext = zext <1 x i16> %load to <1 x i64> @@ -348,6 +581,15 @@ define void @local_zextload_v1i16_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i1 } ; FUNC-LABEL: {{^}}local_sextload_v1i16_to_v1i64: + +; EG: MOV {{[* ]*}}[[FROM:T[0-9]+\.[XYZW]]], KC0[2].Z +; EG: LDS_USHORT_READ_RET {{.*}} [[FROM]] +; EG-DAG: MOV {{[* ]*}}[[TMP:T[0-9]+\.[XYZW]]], OQAP +; EG-DAG: MOV {{[* ]*}}[[TO:T[0-9]+\.[XYZW]]], KC0[2].Y +; EG-DAG: BFE_INT {{[* ]*}}[[DATA:T[0-9]+\.[XYZW]]], {{.*}}, 0.0, literal +; EG-DAG: LDS_WRITE +; EG-DAG: 16 +; EG: LDS_WRITE {{\*?}} [[TO]], [[DATA]] define void @local_sextload_v1i16_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i16> addrspace(3)* %in) #0 { %load = load <1 x i16>, <1 x i16> addrspace(3)* %in %ext = sext <1 x i16> %load to <1 x i64> @@ -356,6 +598,8 @@ define void @local_sextload_v1i16_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i1 } ; FUNC-LABEL: {{^}}local_zextload_v2i16_to_v2i64: + +; EG: LDS_READ_RET define void @local_zextload_v2i16_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(3)* %in %ext = zext <2 x i16> %load to <2 x i64> @@ -364,6 +608,10 @@ define void @local_zextload_v2i16_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i1 } ; FUNC-LABEL: {{^}}local_sextload_v2i16_to_v2i64: + +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: ASHR define void @local_sextload_v2i16_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i16> addrspace(3)* %in) #0 { %load = load <2 x i16>, <2 x i16> addrspace(3)* %in %ext = sext <2 x i16> %load to <2 x i64> @@ -372,6 +620,9 @@ define void @local_sextload_v2i16_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i1 } ; FUNC-LABEL: {{^}}local_zextload_v4i16_to_v4i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v4i16_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i16> addrspace(3)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(3)* %in %ext = zext <4 x i16> %load to <4 x i64> @@ -380,6 +631,13 @@ define void @local_zextload_v4i16_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i1 } ; FUNC-LABEL: {{^}}local_sextload_v4i16_to_v4i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR define void @local_sextload_v4i16_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i16> addrspace(3)* %in) #0 { %load = load <4 x i16>, <4 x i16> addrspace(3)* %in %ext = sext <4 x i16> %load to <4 x i64> @@ -388,6 +646,11 @@ define void @local_sextload_v4i16_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i1 } ; FUNC-LABEL: {{^}}local_zextload_v8i16_to_v8i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v8i16_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(3)* %in %ext = zext <8 x i16> %load to <8 x i64> @@ -396,6 +659,19 @@ define void @local_zextload_v8i16_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i1 } ; FUNC-LABEL: {{^}}local_sextload_v8i16_to_v8i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR define void @local_sextload_v8i16_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i16> addrspace(3)* %in) #0 { %load = load <8 x i16>, <8 x i16> addrspace(3)* %in %ext = sext <8 x i16> %load to <8 x i64> @@ -404,6 +680,15 @@ define void @local_sextload_v8i16_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i1 } ; FUNC-LABEL: {{^}}local_zextload_v16i16_to_v16i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v16i16_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(3)* %in %ext = zext <16 x i16> %load to <16 x i64> @@ -412,6 +697,31 @@ define void @local_zextload_v16i16_to_v16i64(<16 x i64> addrspace(3)* %out, <16 } ; FUNC-LABEL: {{^}}local_sextload_v16i16_to_v16i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR define void @local_sextload_v16i16_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i16> addrspace(3)* %in) #0 { %load = load <16 x i16>, <16 x i16> addrspace(3)* %in %ext = sext <16 x i16> %load to <16 x i64> @@ -420,6 +730,23 @@ define void @local_sextload_v16i16_to_v16i64(<16 x i64> addrspace(3)* %out, <16 } ; FUNC-LABEL: {{^}}local_zextload_v32i16_to_v32i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v32i16_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(3)* %in %ext = zext <32 x i16> %load to <32 x i64> @@ -428,6 +755,55 @@ define void @local_zextload_v32i16_to_v32i64(<32 x i64> addrspace(3)* %out, <32 } ; FUNC-LABEL: {{^}}local_sextload_v32i16_to_v32i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR define void @local_sextload_v32i16_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i16> addrspace(3)* %in) #0 { %load = load <32 x i16>, <32 x i16> addrspace(3)* %in %ext = sext <32 x i16> %load to <32 x i64> diff --git a/test/CodeGen/AMDGPU/load-local-i32.ll b/test/CodeGen/AMDGPU/load-local-i32.ll index d68a8518e2ed..271e563024c0 100644 --- a/test/CodeGen/AMDGPU/load-local-i32.ll +++ b/test/CodeGen/AMDGPU/load-local-i32.ll @@ -36,7 +36,7 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v4i32: -; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1{{$}} +; GCN: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} define void @local_load_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> addrspace(3)* %in) #0 { entry: @@ -46,8 +46,8 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v8i32: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:3 offset1:2{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} define void @local_load_v8i32(<8 x i32> addrspace(3)* %out, <8 x i32> addrspace(3)* %in) #0 { entry: %ld = load <8 x i32>, <8 x i32> addrspace(3)* %in @@ -56,10 +56,14 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v16i32: -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:3 offset1:4{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:5 offset1:6{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7{{$}} -; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:1 offset1:2{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:6 offset1:7{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:4 offset1:5{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:2 offset1:3{{$}} +; GCN-DAG: ds_read2_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:1{{$}} +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:6 offset1:7 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:4 offset1:5 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3 +; GCN-DAG: ds_write2_b64 v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}} offset1:1 define void @local_load_v16i32(<16 x i32> addrspace(3)* %out, <16 x i32> addrspace(3)* %in) #0 { entry: %ld = load <16 x i32>, <16 x i32> addrspace(3)* %in diff --git a/test/CodeGen/AMDGPU/load-local-i8.ll b/test/CodeGen/AMDGPU/load-local-i8.ll index be865b078d74..02b59e89c3f3 100644 --- a/test/CodeGen/AMDGPU/load-local-i8.ll +++ b/test/CodeGen/AMDGPU/load-local-i8.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s @@ -64,8 +64,8 @@ entry: } ; FUNC-LABEL: {{^}}local_load_v16i8: -; GCN: ds_read2_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}} offset0:1{{$}} -; GCN: ds_write2_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:{{[0-9]+}}], v[{{[0-9]+}}:[[HI]]{{\]}} offset0:1{{$}} +; GCN: ds_read2_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:1{{$}} +; GCN: ds_write2_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:{{[0-9]+}}], v[{{[0-9]+}}:[[HI]]{{\]}} offset1:1{{$}} ; EG: LDS_READ_RET ; EG: LDS_READ_RET @@ -106,6 +106,8 @@ define void @local_sextload_i8_to_i32(i32 addrspace(3)* %out, i8 addrspace(3)* % } ; FUNC-LABEL: {{^}}local_zextload_v1i8_to_v1i32: + +; EG: LDS_UBYTE_READ_RET define void @local_zextload_v1i8_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = zext <1 x i8> %load to <1 x i32> @@ -114,6 +116,9 @@ define void @local_zextload_v1i8_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i32: + +; EG: LDS_UBYTE_READ_RET +; EG: BFE_INT define void @local_sextload_v1i8_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = sext <1 x i8> %load to <1 x i32> @@ -124,8 +129,7 @@ define void @local_sextload_v1i8_to_v1i32(<1 x i32> addrspace(3)* %out, <1 x i8> ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i32: ; GCN: ds_read_u16 -; EG: LDS_UBYTE_READ_RET -; EG: LDS_UBYTE_READ_RET +; EG: LDS_USHORT_READ_RET define void @local_zextload_v2i8_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(3)* %in %ext = zext <2 x i8> %load to <2 x i32> @@ -137,11 +141,19 @@ define void @local_zextload_v2i8_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i8> ; GCN-NOT: s_wqm_b64 ; GCN: s_mov_b32 m0 ; GCN: ds_read_u16 -; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 -; GCN-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 +; FIXME: Need to optimize this sequence to avoid extra shift on VI. +; t23: i16 = srl t39, Constant:i32<8> +; t31: i32 = any_extend t23 +; t33: i32 = sign_extend_inreg t31, ValueType:ch:i8 + +; SI-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; SI-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 -; EG-DAG: LDS_UBYTE_READ_RET -; EG-DAG: LDS_UBYTE_READ_RET +; VI-DAG: v_lshrrev_b16_e32 [[SHIFT:v[0-9]+]], 8, v{{[0-9]+}} +; VI-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 +; VI-DAG: v_bfe_i32 v{{[0-9]+}}, [[SHIFT]], 0, 8 + +; EG: LDS_USHORT_READ_RET ; EG-DAG: BFE_INT ; EG-DAG: BFE_INT define void @local_sextload_v2i8_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { @@ -154,9 +166,12 @@ define void @local_sextload_v2i8_to_v2i32(<2 x i32> addrspace(3)* %out, <2 x i8> ; FUNC-LABEL: {{^}}local_zextload_v3i8_to_v3i32: ; GCN: ds_read_b32 -; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; SI-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 8, 8 +; VI-DAG: v_lshrrev_b16_e32 v{{[0-9]+}}, 8, {{v[0-9]+}} ; GCN-DAG: v_bfe_u32 v{{[0-9]+}}, v{{[0-9]+}}, 16, 8 ; GCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 0xff, + +; EG: LDS_READ_RET define void @local_zextload_v3i8_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i8> addrspace(3)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(3)* %in @@ -178,6 +193,10 @@ entry: ; GCN-DAG: ds_write_b64 ; GCN-DAG: ds_write_b32 +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v3i8_to_v3i32(<3 x i32> addrspace(3)* %out, <3 x i8> addrspace(3)* %in) #0 { entry: %ld = load <3 x i8>, <3 x i8> addrspace(3)* %in @@ -191,10 +210,10 @@ entry: ; GCN: s_mov_b32 m0 ; GCN: ds_read_b32 -; EG: LDS_UBYTE_READ_RET -; EG: LDS_UBYTE_READ_RET -; EG: LDS_UBYTE_READ_RET -; EG: LDS_UBYTE_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT define void @local_zextload_v4i8_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(3)* %in %ext = zext <4 x i8> %load to <4 x i32> @@ -207,10 +226,7 @@ define void @local_zextload_v4i8_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i8> ; GCN: s_mov_b32 m0 ; GCN: ds_read_b32 -; EG-DAG: LDS_UBYTE_READ_RET -; EG-DAG: LDS_UBYTE_READ_RET -; EG-DAG: LDS_UBYTE_READ_RET -; EG-DAG: LDS_UBYTE_READ_RET +; EG-DAG: LDS_READ_RET ; EG-DAG: BFE_INT ; EG-DAG: BFE_INT ; EG-DAG: BFE_INT @@ -223,6 +239,15 @@ define void @local_sextload_v4i8_to_v4i32(<4 x i32> addrspace(3)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT define void @local_zextload_v8i8_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = zext <8 x i8> %load to <8 x i32> @@ -231,6 +256,17 @@ define void @local_zextload_v8i8_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v8i8_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = sext <8 x i8> %load to <8 x i32> @@ -239,6 +275,23 @@ define void @local_sextload_v8i8_to_v8i32(<8 x i32> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT +; EG-DAG: BFE_UINT define void @local_zextload_v16i8_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = zext <16 x i8> %load to <16 x i32> @@ -247,6 +300,27 @@ define void @local_zextload_v16i8_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v16i8_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = sext <16 x i8> %load to <16 x i32> @@ -255,6 +329,15 @@ define void @local_sextload_v16i8_to_v16i32(<16 x i32> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET define void @local_zextload_v32i8_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = zext <32 x i8> %load to <32 x i32> @@ -263,6 +346,15 @@ define void @local_zextload_v32i8_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x } ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET define void @local_sextload_v32i8_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = sext <32 x i8> %load to <32 x i32> @@ -271,6 +363,23 @@ define void @local_sextload_v32i8_to_v32i32(<32 x i32> addrspace(3)* %out, <32 x } ; FUNC-LABEL: {{^}}local_zextload_v64i8_to_v64i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET define void @local_zextload_v64i8_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i8> addrspace(3)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(3)* %in %ext = zext <64 x i8> %load to <64 x i32> @@ -279,6 +388,23 @@ define void @local_zextload_v64i8_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x } ; FUNC-LABEL: {{^}}local_sextload_v64i8_to_v64i32: + +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET +; EG-DAG: LDS_READ_RET define void @local_sextload_v64i8_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x i8> addrspace(3)* %in) #0 { %load = load <64 x i8>, <64 x i8> addrspace(3)* %in %ext = sext <64 x i8> %load to <64 x i32> @@ -290,6 +416,10 @@ define void @local_sextload_v64i8_to_v64i32(<64 x i32> addrspace(3)* %out, <64 x ; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0{{$}} ; GCN-DAG: ds_read_u8 v[[LO:[0-9]+]], ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]] + +; EG: LDS_UBYTE_READ_RET +; EG: MOV {{.*}}, literal +; EG: 0.0 define void @local_zextload_i8_to_i64(i64 addrspace(3)* %out, i8 addrspace(3)* %in) #0 { %a = load i8, i8 addrspace(3)* %in %ext = zext i8 %a to i64 @@ -302,6 +432,11 @@ define void @local_zextload_i8_to_i64(i64 addrspace(3)* %out, i8 addrspace(3)* % ; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] ; GCN: ds_write_b64 v{{[0-9]+}}, v{{\[}}[[LO]]:[[HI]]{{\]}} + +; EG: LDS_UBYTE_READ_RET +; EG: ASHR +; TODO: why not 7? +; EG: 31 define void @local_sextload_i8_to_i64(i64 addrspace(3)* %out, i8 addrspace(3)* %in) #0 { %a = load i8, i8 addrspace(3)* %in %ext = sext i8 %a to i64 @@ -310,6 +445,11 @@ define void @local_sextload_i8_to_i64(i64 addrspace(3)* %out, i8 addrspace(3)* % } ; FUNC-LABEL: {{^}}local_zextload_v1i8_to_v1i64: + +; EG: LDS_UBYTE_READ_RET +; EG: MOV {{.*}}, literal +; TODO: merge? +; EG: 0.0 define void @local_zextload_v1i8_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = zext <1 x i8> %load to <1 x i64> @@ -318,6 +458,11 @@ define void @local_zextload_v1i8_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i64: + +; EG: LDS_UBYTE_READ_RET +; EG: ASHR +; TODO: why not 7? +; EG: 31 define void @local_sextload_v1i8_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = sext <1 x i8> %load to <1 x i64> @@ -326,6 +471,8 @@ define void @local_sextload_v1i8_to_v1i64(<1 x i64> addrspace(3)* %out, <1 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i64: + +; EG: LDS_USHORT_READ_RET define void @local_zextload_v2i8_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(3)* %in %ext = zext <2 x i8> %load to <2 x i64> @@ -334,6 +481,10 @@ define void @local_zextload_v2i8_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v2i8_to_v2i64: + +; EG: LDS_USHORT_READ_RET +; EG: BFE_INT +; EG: BFE_INT define void @local_sextload_v2i8_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(3)* %in %ext = sext <2 x i8> %load to <2 x i64> @@ -342,6 +493,8 @@ define void @local_sextload_v2i8_to_v2i64(<2 x i64> addrspace(3)* %out, <2 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v4i8_to_v4i64: + +; EG: LDS_READ_RET define void @local_zextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(3)* %in %ext = zext <4 x i8> %load to <4 x i64> @@ -350,6 +503,8 @@ define void @local_zextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v4i8_to_v4i64: + +; EG: LDS_READ_RET define void @local_sextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(3)* %in %ext = sext <4 x i8> %load to <4 x i64> @@ -358,6 +513,9 @@ define void @local_sextload_v4i8_to_v4i64(<4 x i64> addrspace(3)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v8i8_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = zext <8 x i8> %load to <8 x i64> @@ -366,6 +524,18 @@ define void @local_zextload_v8i8_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT define void @local_sextload_v8i8_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = sext <8 x i8> %load to <8 x i64> @@ -374,6 +544,11 @@ define void @local_sextload_v8i8_to_v8i64(<8 x i64> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v16i8_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = zext <16 x i8> %load to <16 x i64> @@ -382,6 +557,11 @@ define void @local_zextload_v16i8_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_sextload_v16i8_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = sext <16 x i8> %load to <16 x i64> @@ -390,6 +570,15 @@ define void @local_sextload_v16i8_to_v16i64(<16 x i64> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_zextload_v32i8_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = zext <32 x i8> %load to <32 x i64> @@ -398,6 +587,15 @@ define void @local_zextload_v32i8_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x } ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i64: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET define void @local_sextload_v32i8_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = sext <32 x i8> %load to <32 x i64> @@ -424,6 +622,9 @@ define void @local_sextload_v32i8_to_v32i64(<32 x i64> addrspace(3)* %out, <32 x ; FUNC-LABEL: {{^}}local_zextload_i8_to_i16: ; GCN: ds_read_u8 v[[VAL:[0-9]+]], ; GCN: ds_write_b16 v[[VAL:[0-9]+]] + +; EG: LDS_UBYTE_READ_RET +; EG: LDS_SHORT_WRITE define void @local_zextload_i8_to_i16(i16 addrspace(3)* %out, i8 addrspace(3)* %in) #0 { %a = load i8, i8 addrspace(3)* %in %ext = zext i8 %a to i16 @@ -434,6 +635,10 @@ define void @local_zextload_i8_to_i16(i16 addrspace(3)* %out, i8 addrspace(3)* % ; FUNC-LABEL: {{^}}local_sextload_i8_to_i16: ; GCN: ds_read_i8 v[[VAL:[0-9]+]], ; GCN: ds_write_b16 v{{[0-9]+}}, v[[VAL]] + +; EG: LDS_UBYTE_READ_RET +; EG: BFE_INT +; EG: LDS_SHORT_WRITE define void @local_sextload_i8_to_i16(i16 addrspace(3)* %out, i8 addrspace(3)* %in) #0 { %a = load i8, i8 addrspace(3)* %in %ext = sext i8 %a to i16 @@ -442,6 +647,9 @@ define void @local_sextload_i8_to_i16(i16 addrspace(3)* %out, i8 addrspace(3)* % } ; FUNC-LABEL: {{^}}local_zextload_v1i8_to_v1i16: + +; EG: LDS_UBYTE_READ_RET +; EG: LDS_SHORT_WRITE define void @local_zextload_v1i8_to_v1i16(<1 x i16> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = zext <1 x i8> %load to <1 x i16> @@ -450,6 +658,10 @@ define void @local_zextload_v1i8_to_v1i16(<1 x i16> addrspace(3)* %out, <1 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v1i8_to_v1i16: + +; EG: LDS_UBYTE_READ_RET +; EG: BFE_INT +; EG: LDS_SHORT_WRITE define void @local_sextload_v1i8_to_v1i16(<1 x i16> addrspace(3)* %out, <1 x i8> addrspace(3)* %in) #0 { %load = load <1 x i8>, <1 x i8> addrspace(3)* %in %ext = sext <1 x i8> %load to <1 x i16> @@ -458,6 +670,9 @@ define void @local_sextload_v1i8_to_v1i16(<1 x i16> addrspace(3)* %out, <1 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v2i8_to_v2i16: + +; EG: LDS_USHORT_READ_RET +; EG: LDS_WRITE define void @local_zextload_v2i8_to_v2i16(<2 x i16> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(3)* %in %ext = zext <2 x i8> %load to <2 x i16> @@ -466,6 +681,11 @@ define void @local_zextload_v2i8_to_v2i16(<2 x i16> addrspace(3)* %out, <2 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v2i8_to_v2i16: + +; EG: LDS_USHORT_READ_RET +; EG: BFE_INT +; EG: BFE_INT +; EG: LDS_WRITE define void @local_sextload_v2i8_to_v2i16(<2 x i16> addrspace(3)* %out, <2 x i8> addrspace(3)* %in) #0 { %load = load <2 x i8>, <2 x i8> addrspace(3)* %in %ext = sext <2 x i8> %load to <2 x i16> @@ -474,6 +694,10 @@ define void @local_sextload_v2i8_to_v2i16(<2 x i16> addrspace(3)* %out, <2 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v4i8_to_v4i16: + +; EG: LDS_READ_RET +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_zextload_v4i8_to_v4i16(<4 x i16> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(3)* %in %ext = zext <4 x i8> %load to <4 x i16> @@ -482,6 +706,14 @@ define void @local_zextload_v4i8_to_v4i16(<4 x i16> addrspace(3)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v4i8_to_v4i16: + +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_sextload_v4i8_to_v4i16(<4 x i16> addrspace(3)* %out, <4 x i8> addrspace(3)* %in) #0 { %load = load <4 x i8>, <4 x i8> addrspace(3)* %in %ext = sext <4 x i8> %load to <4 x i16> @@ -490,6 +722,13 @@ define void @local_sextload_v4i8_to_v4i16(<4 x i16> addrspace(3)* %out, <4 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v8i8_to_v8i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_zextload_v8i8_to_v8i16(<8 x i16> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = zext <8 x i8> %load to <8 x i16> @@ -498,6 +737,21 @@ define void @local_zextload_v8i8_to_v8i16(<8 x i16> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_sextload_v8i8_to_v8i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_sextload_v8i8_to_v8i16(<8 x i16> addrspace(3)* %out, <8 x i8> addrspace(3)* %in) #0 { %load = load <8 x i8>, <8 x i8> addrspace(3)* %in %ext = sext <8 x i8> %load to <8 x i16> @@ -506,6 +760,19 @@ define void @local_sextload_v8i8_to_v8i16(<8 x i16> addrspace(3)* %out, <8 x i8> } ; FUNC-LABEL: {{^}}local_zextload_v16i8_to_v16i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_zextload_v16i8_to_v16i16(<16 x i16> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = zext <16 x i8> %load to <16 x i16> @@ -514,6 +781,35 @@ define void @local_zextload_v16i8_to_v16i16(<16 x i16> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_sextload_v16i8_to_v16i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_sextload_v16i8_to_v16i16(<16 x i16> addrspace(3)* %out, <16 x i8> addrspace(3)* %in) #0 { %load = load <16 x i8>, <16 x i8> addrspace(3)* %in %ext = sext <16 x i8> %load to <16 x i16> @@ -522,6 +818,31 @@ define void @local_sextload_v16i8_to_v16i16(<16 x i16> addrspace(3)* %out, <16 x } ; FUNC-LABEL: {{^}}local_zextload_v32i8_to_v32i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_zextload_v32i8_to_v32i16(<32 x i16> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = zext <32 x i8> %load to <32 x i16> @@ -530,6 +851,63 @@ define void @local_zextload_v32i8_to_v32i16(<32 x i16> addrspace(3)* %out, <32 x } ; FUNC-LABEL: {{^}}local_sextload_v32i8_to_v32i16: + +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG: LDS_READ_RET +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: BFE_INT +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG-DAG: ASHR +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE define void @local_sextload_v32i8_to_v32i16(<32 x i16> addrspace(3)* %out, <32 x i8> addrspace(3)* %in) #0 { %load = load <32 x i8>, <32 x i8> addrspace(3)* %in %ext = sext <32 x i8> %load to <32 x i16> diff --git a/test/CodeGen/AMDGPU/local-64.ll b/test/CodeGen/AMDGPU/local-64.ll index f63d6e08ef73..5cabd4dd81be 100644 --- a/test/CodeGen/AMDGPU/local-64.ll +++ b/test/CodeGen/AMDGPU/local-64.ll @@ -122,7 +122,7 @@ define void @local_f64_store_0_offset(double addrspace(3)* %out) nounwind { ; BOTH-LABEL: {{^}}local_v2i64_store: ; BOTH-NOT: ADD -; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:15 offset1:14 +; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:14 offset1:15 ; BOTH: s_endpgm define void @local_v2i64_store(<2 x i64> addrspace(3)* %out) nounwind { %gep = getelementptr <2 x i64>, <2 x i64> addrspace(3)* %out, i32 7 @@ -132,7 +132,7 @@ define void @local_v2i64_store(<2 x i64> addrspace(3)* %out) nounwind { ; BOTH-LABEL: {{^}}local_v2i64_store_0_offset: ; BOTH-NOT: ADD -; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:1 +; BOTH: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1 ; BOTH: s_endpgm define void @local_v2i64_store_0_offset(<2 x i64> addrspace(3)* %out) nounwind { store <2 x i64> <i64 1234, i64 1234>, <2 x i64> addrspace(3)* %out, align 16 @@ -141,8 +141,8 @@ define void @local_v2i64_store_0_offset(<2 x i64> addrspace(3)* %out) nounwind { ; BOTH-LABEL: {{^}}local_v4i64_store: ; BOTH-NOT: ADD -; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:31 offset1:30 -; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:29 offset1:28 +; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:30 offset1:31 +; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:28 offset1:29 ; BOTH: s_endpgm define void @local_v4i64_store(<4 x i64> addrspace(3)* %out) nounwind { %gep = getelementptr <4 x i64>, <4 x i64> addrspace(3)* %out, i32 7 @@ -152,8 +152,8 @@ define void @local_v4i64_store(<4 x i64> addrspace(3)* %out) nounwind { ; BOTH-LABEL: {{^}}local_v4i64_store_0_offset: ; BOTH-NOT: ADD -; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:3 offset1:2 -; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:1 +; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset0:2 offset1:3 +; BOTH-DAG: ds_write2_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}} offset1:1 ; BOTH: s_endpgm define void @local_v4i64_store_0_offset(<4 x i64> addrspace(3)* %out) nounwind { store <4 x i64> <i64 1234, i64 1234, i64 1234, i64 1234>, <4 x i64> addrspace(3)* %out, align 16 diff --git a/test/CodeGen/AMDGPU/local-memory.amdgcn.ll b/test/CodeGen/AMDGPU/local-memory.amdgcn.ll index f6c0e3c62390..a57e4f595322 100644 --- a/test/CodeGen/AMDGPU/local-memory.amdgcn.ll +++ b/test/CodeGen/AMDGPU/local-memory.amdgcn.ll @@ -44,8 +44,7 @@ entry: ; GCN-LABEL: {{^}}local_memory_two_objects: ; GCN: v_lshlrev_b32_e32 [[ADDRW:v[0-9]+]], 2, v0 -; CI-DAG: ds_write_b32 [[ADDRW]], {{v[0-9]*}} offset:16 -; CI-DAG: ds_write_b32 [[ADDRW]], {{v[0-9]*$}} +; CI-DAG: ds_write2_b32 [[ADDRW]], {{v[0-9]+}}, {{v[0-9]+}} offset1:4 ; SI: v_add_i32_e32 [[ADDRW_OFF:v[0-9]+]], vcc, 16, [[ADDRW]] diff --git a/test/CodeGen/AMDGPU/local-stack-slot-bug.ll b/test/CodeGen/AMDGPU/local-stack-slot-bug.ll index 6e6f289f5d6d..d49fa2bf48a7 100644 --- a/test/CodeGen/AMDGPU/local-stack-slot-bug.ll +++ b/test/CodeGen/AMDGPU/local-stack-slot-bug.ll @@ -6,11 +6,13 @@ ; from https://bugs.freedesktop.org/show_bug.cgi?id=96602 ; ; CHECK-LABEL: {{^}}main: -; CHECK: v_lshlrev_b32_e32 [[BYTES:v[0-9]+]], 2, v0 -; CHECK: v_mov_b32_e32 [[HI_CONST:v[0-9]+]], 0x200 -; CHECK: v_mov_b32_e32 [[LO_CONST:v[0-9]+]], 0 -; CHECK: v_add_i32_e32 [[HI_OFF:v[0-9]+]], vcc, [[BYTES]], [[HI_CONST]] -; CHECK: v_add_i32_e32 [[LO_OFF:v[0-9]+]], vcc, [[BYTES]], [[LO_CONST]] + +; CHECK-DAG: v_lshlrev_b32_e32 [[BYTES:v[0-9]+]], 2, v0 +; CHECK-DAG: v_add_i32_e32 [[HI_OFF:v[0-9]+]], vcc, 0x200, [[BYTES]] + +; TODO: add 0? +; CHECK-DAG: v_add_i32_e32 [[LO_OFF:v[0-9]+]], vcc, 0, [[BYTES]] + ; CHECK: buffer_load_dword {{v[0-9]+}}, [[LO_OFF]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen ; CHECK: buffer_load_dword {{v[0-9]+}}, [[HI_OFF]], {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen define amdgpu_ps float @main(i32 %idx) { diff --git a/test/CodeGen/AMDGPU/local-stack-slot-offset.ll b/test/CodeGen/AMDGPU/local-stack-slot-offset.ll new file mode 100644 index 000000000000..256f6e87c84b --- /dev/null +++ b/test/CodeGen/AMDGPU/local-stack-slot-offset.ll @@ -0,0 +1,35 @@ +;RUN: llc < %s -march=amdgcn -mcpu=verde -mattr=+vgpr-spilling -mattr=-promote-alloca -verify-machineinstrs | FileCheck %s -check-prefix=CHECK +;RUN: llc < %s -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -mattr=-promote-alloca -verify-machineinstrs | FileCheck %s -check-prefix=CHECK + +; Allocate two stack slots of 2052 bytes each requiring a total of 4104 bytes. +; Extracting the last element of each does not fit into the offset field of +; MUBUF instructions, so a new base register is needed. This used to not +; happen, leading to an assertion. + +; CHECK-LABEL: {{^}}main: +; CHECK: buffer_store_dword +; CHECK: buffer_store_dword +; CHECK: buffer_load_dword +; CHECK: buffer_load_dword +define amdgpu_gs float @main(float %v1, float %v2, i32 %idx1, i32 %idx2) { +main_body: + %m1 = alloca [513 x float] + %m2 = alloca [513 x float] + + %gep1.store = getelementptr [513 x float], [513 x float]* %m1, i32 0, i32 %idx1 + store float %v1, float* %gep1.store + + %gep2.store = getelementptr [513 x float], [513 x float]* %m2, i32 0, i32 %idx2 + store float %v2, float* %gep2.store + +; This used to use a base reg equal to 0. + %gep1.load = getelementptr [513 x float], [513 x float]* %m1, i32 0, i32 0 + %out1 = load float, float* %gep1.load + +; This used to attempt to re-use the base reg at 0, generating an out-of-bounds instruction offset. + %gep2.load = getelementptr [513 x float], [513 x float]* %m2, i32 0, i32 512 + %out2 = load float, float* %gep2.load + + %r = fadd float %out1, %out2 + ret float %r +} diff --git a/test/CodeGen/AMDGPU/loop_break.ll b/test/CodeGen/AMDGPU/loop_break.ll new file mode 100644 index 000000000000..82564b8bb28d --- /dev/null +++ b/test/CodeGen/AMDGPU/loop_break.ll @@ -0,0 +1,70 @@ +; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow %s | FileCheck -check-prefix=OPT %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; Uses llvm.amdgcn.break + +; OPT-LABEL: @break_loop( +; OPT: bb1: +; OPT: call i64 @llvm.amdgcn.break(i64 +; OPT-NEXT: br i1 %cmp0, label %bb4, label %Flow + +; OPT: bb4: +; OPT: load volatile +; OPT: xor i1 %cmp1 +; OPT: call i64 @llvm.amdgcn.if.break( +; OPT: br label %Flow + +; OPT: Flow: +; OPT: call i1 @llvm.amdgcn.loop(i64 +; OPT: br i1 %{{[0-9]+}}, label %bb9, label %bb1 + +; OPT: bb9: +; OPT: call void @llvm.amdgcn.end.cf(i64 + +; TODO: Can remove exec fixes in return block +; GCN-LABEL: {{^}}break_loop: +; GCN: s_mov_b64 [[INITMASK:s\[[0-9]+:[0-9]+\]]], 0{{$}} + +; GCN: [[LOOP_ENTRY:BB[0-9]+_[0-9]+]]: ; %bb1 +; GCN: s_or_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[INITMASK]] +; GCN: s_cmp_gt_i32 s{{[0-9]+}}, -1 +; GCN-NEXT: s_cbranch_scc1 [[FLOW:BB[0-9]+_[0-9]+]] + +; GCN: ; BB#2: ; %bb4 +; GCN: buffer_load_dword +; GCN: v_cmp_ge_i32_e32 vcc, +; GCN: s_or_b64 [[MASK]], vcc, [[INITMASK]] + +; GCN: [[FLOW]]: +; GCN: s_mov_b64 [[INITMASK]], [[MASK]] +; GCN: s_andn2_b64 exec, exec, [[MASK]] +; GCN-NEXT: s_cbranch_execnz [[LOOP_ENTRY]] + +; GCN: ; BB#4: ; %bb9 +; GCN-NEXT: s_or_b64 exec, exec, [[MASK]] +; GCN-NEXT: s_endpgm +define void @break_loop(i32 %arg) #0 { +bb: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %tmp = sub i32 %id, %arg + br label %bb1 + +bb1: + %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %bb4 ] + %lsr.iv.next = add i32 %lsr.iv, 1 + %cmp0 = icmp slt i32 %lsr.iv.next, 0 + br i1 %cmp0, label %bb4, label %bb9 + +bb4: + %load = load volatile i32, i32 addrspace(1)* undef, align 4 + %cmp1 = icmp slt i32 %tmp, %load + br i1 %cmp1, label %bb1, label %bb9 + +bb9: + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/m0-spill.ll b/test/CodeGen/AMDGPU/m0-spill.ll deleted file mode 100644 index 2427c8de34f8..000000000000 --- a/test/CodeGen/AMDGPU/m0-spill.ll +++ /dev/null @@ -1,35 +0,0 @@ -; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s - -@lds = external addrspace(3) global [64 x float] - -; CHECK-LABEL: {{^}}main: -; CHECK-NOT: v_readlane_b32 m0 -define amdgpu_ps void @main(<16 x i8> addrspace(2)* inreg, <16 x i8> addrspace(2)* inreg, <32 x i8> addrspace(2)* inreg, i32 inreg) { -main_body: - %4 = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %3) - %cmp = fcmp ueq float 0.0, %4 - br i1 %cmp, label %if, label %else - -if: - %lds_ptr = getelementptr [64 x float], [64 x float] addrspace(3)* @lds, i32 0, i32 0 - %lds_data = load float, float addrspace(3)* %lds_ptr - br label %endif - -else: - %interp = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %3) - br label %endif - -endif: - %export = phi float [%lds_data, %if], [%interp, %else] - %5 = call i32 @llvm.SI.packf16(float %export, float %export) - %6 = bitcast i32 %5 to float - call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %6, float %6, float %6, float %6) - ret void -} - -declare float @llvm.SI.fs.constant(i32, i32, i32) readnone - -declare i32 @llvm.SI.packf16(float, float) readnone - -declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) diff --git a/test/CodeGen/AMDGPU/mad-sub.ll b/test/CodeGen/AMDGPU/mad-sub.ll deleted file mode 100644 index 7fcfe7f53f06..000000000000 --- a/test/CodeGen/AMDGPU/mad-sub.ll +++ /dev/null @@ -1,215 +0,0 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s - -declare i32 @llvm.amdgcn.workitem.id.x() #0 -declare float @llvm.fabs.f32(float) #0 - -; FUNC-LABEL: {{^}}mad_sub_f32: -; SI: buffer_load_dword [[REGA:v[0-9]+]] -; SI: buffer_load_dword [[REGB:v[0-9]+]] -; SI: buffer_load_dword [[REGC:v[0-9]+]] -; SI: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -[[REGC]] -; SI: buffer_store_dword [[RESULT]] -define void @mad_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %mul = fmul float %a, %b - %sub = fsub float %mul, %c - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}mad_sub_inv_f32: -; SI: buffer_load_dword [[REGA:v[0-9]+]] -; SI: buffer_load_dword [[REGB:v[0-9]+]] -; SI: buffer_load_dword [[REGC:v[0-9]+]] -; SI: v_mad_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], [[REGC]] -; SI: buffer_store_dword [[RESULT]] -define void @mad_sub_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %mul = fmul float %a, %b - %sub = fsub float %c, %mul - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}mad_sub_f64: -; SI: v_mul_f64 -; SI: v_add_f64 -define void @mad_sub_f64(double addrspace(1)* noalias nocapture %out, double addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr double, double addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr double, double addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr double, double addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr double, double addrspace(1)* %out, i64 %tid.ext - %a = load volatile double, double addrspace(1)* %gep0, align 8 - %b = load volatile double, double addrspace(1)* %gep1, align 8 - %c = load volatile double, double addrspace(1)* %gep2, align 8 - %mul = fmul double %a, %b - %sub = fsub double %mul, %c - store double %sub, double addrspace(1)* %outgep, align 8 - ret void -} - -; FUNC-LABEL: {{^}}mad_sub_fabs_f32: -; SI: buffer_load_dword [[REGA:v[0-9]+]] -; SI: buffer_load_dword [[REGB:v[0-9]+]] -; SI: buffer_load_dword [[REGC:v[0-9]+]] -; SI: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], [[REGB]], -|[[REGC]]| -; SI: buffer_store_dword [[RESULT]] -define void @mad_sub_fabs_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %c.abs = call float @llvm.fabs.f32(float %c) #0 - %mul = fmul float %a, %b - %sub = fsub float %mul, %c.abs - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}mad_sub_fabs_inv_f32: -; SI: buffer_load_dword [[REGA:v[0-9]+]] -; SI: buffer_load_dword [[REGB:v[0-9]+]] -; SI: buffer_load_dword [[REGC:v[0-9]+]] -; SI: v_mad_f32 [[RESULT:v[0-9]+]], -[[REGA]], [[REGB]], |[[REGC]]| -; SI: buffer_store_dword [[RESULT]] -define void @mad_sub_fabs_inv_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %c.abs = call float @llvm.fabs.f32(float %c) #0 - %mul = fmul float %a, %b - %sub = fsub float %c.abs, %mul - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}neg_neg_mad_f32: -; SI: v_mac_f32_e32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} -define void @neg_neg_mad_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %nega = fsub float -0.000000e+00, %a - %negb = fsub float -0.000000e+00, %b - %mul = fmul float %nega, %negb - %sub = fadd float %mul, %c - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}mad_fabs_sub_f32: -; SI: buffer_load_dword [[REGA:v[0-9]+]] -; SI: buffer_load_dword [[REGB:v[0-9]+]] -; SI: buffer_load_dword [[REGC:v[0-9]+]] -; SI: v_mad_f32 [[RESULT:v[0-9]+]], [[REGA]], |[[REGB]]|, -[[REGC]] -; SI: buffer_store_dword [[RESULT]] -define void @mad_fabs_sub_f32(float addrspace(1)* noalias nocapture %out, float addrspace(1)* noalias nocapture readonly %ptr) #1 { - %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0 - %tid.ext = sext i32 %tid to i64 - %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 %tid.ext - %add1 = add i64 %tid.ext, 1 - %gep1 = getelementptr float, float addrspace(1)* %ptr, i64 %add1 - %add2 = add i64 %tid.ext, 2 - %gep2 = getelementptr float, float addrspace(1)* %ptr, i64 %add2 - %outgep = getelementptr float, float addrspace(1)* %out, i64 %tid.ext - %a = load volatile float, float addrspace(1)* %gep0, align 4 - %b = load volatile float, float addrspace(1)* %gep1, align 4 - %c = load volatile float, float addrspace(1)* %gep2, align 4 - %b.abs = call float @llvm.fabs.f32(float %b) #0 - %mul = fmul float %a, %b.abs - %sub = fsub float %mul, %c - store float %sub, float addrspace(1)* %outgep, align 4 - ret void -} - -; FUNC-LABEL: {{^}}fsub_c_fadd_a_a: -; SI-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; SI-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; SI: v_mac_f32_e32 [[R2]], -2.0, [[R1]] -; SI: buffer_store_dword [[R2]] -define void @fsub_c_fadd_a_a(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %add = fadd float %r1, %r1 - %r3 = fsub float %r2, %add - - store float %r3, float addrspace(1)* %gep.out - ret void -} - -; FUNC-LABEL: {{^}}fsub_fadd_a_a_c: -; SI-DAG: buffer_load_dword [[R1:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} -; SI-DAG: buffer_load_dword [[R2:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; SI: v_mad_f32 [[RESULT:v[0-9]+]], 2.0, [[R1]], -[[R2]] -; SI: buffer_store_dword [[RESULT]] -define void @fsub_fadd_a_a_c(float addrspace(1)* %out, float addrspace(1)* %in) { - %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone - %gep.0 = getelementptr float, float addrspace(1)* %out, i32 %tid - %gep.1 = getelementptr float, float addrspace(1)* %gep.0, i32 1 - %gep.out = getelementptr float, float addrspace(1)* %out, i32 %tid - - %r1 = load volatile float, float addrspace(1)* %gep.0 - %r2 = load volatile float, float addrspace(1)* %gep.1 - - %add = fadd float %r1, %r1 - %r3 = fsub float %add, %r2 - - store float %r3, float addrspace(1)* %gep.out - ret void -} - -attributes #0 = { nounwind readnone } -attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/mad_uint24.ll b/test/CodeGen/AMDGPU/mad_uint24.ll index 72c6b2b26173..214534710653 100644 --- a/test/CodeGen/AMDGPU/mad_uint24.ll +++ b/test/CodeGen/AMDGPU/mad_uint24.ll @@ -1,11 +1,15 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC ; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=EG --check-prefix=FUNC +; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC +; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=VI --check-prefix=FUNC +; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck %s --check-prefix=VI --check-prefix=FUNC + +declare i32 @llvm.r600.read.tidig.x() nounwind readnone ; FUNC-LABEL: {{^}}u32_mad24: ; EG: MULADD_UINT24 ; SI: v_mad_u32_u24 +; VI: v_mad_u32_u24 define void @u32_mad24(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { entry: @@ -25,9 +29,9 @@ entry: ; The result must be sign-extended ; EG: BFE_INT {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[MAD_CHAN]], 0.0, literal.x ; EG: 16 -; SI: v_mad_u32_u24 [[MAD:v[0-9]]], {{[sv][0-9], [sv][0-9]}} -; SI: v_bfe_i32 v{{[0-9]}}, [[MAD]], 0, 16 - +; FIXME: Should be using scalar instructions here. +; GCN: v_mad_u32_u24 [[MAD:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; GCN: v_bfe_i32 v{{[0-9]}}, [[MAD]], 0, 16 define void @i16_mad24(i32 addrspace(1)* %out, i16 %a, i16 %b, i16 %c) { entry: %0 = mul i16 %a, %b @@ -37,14 +41,14 @@ entry: ret void } +; FIXME: Need to handle non-uniform case for function below (load without gep). ; FUNC-LABEL: {{^}}i8_mad24: ; EG: MULADD_UINT24 {{[* ]*}}T{{[0-9]}}.[[MAD_CHAN:[XYZW]]] ; The result must be sign-extended ; EG: BFE_INT {{[* ]*}}T{{[0-9]\.[XYZW]}}, PV.[[MAD_CHAN]], 0.0, literal.x ; EG: 8 -; SI: v_mad_u32_u24 [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} -; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8 - +; GCN: v_mad_u32_u24 [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; GCN: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8 define void @i8_mad24(i32 addrspace(1)* %out, i8 %a, i8 %b, i8 %c) { entry: %0 = mul i8 %a, %b @@ -74,3 +78,63 @@ entry: store i32 %4, i32 addrspace(1)* %out ret void } + +; FUNC-LABEL: {{^}}extra_and: +; SI-NOT: v_and +; SI: v_mad_u32_u24 +; SI: v_mad_u32_u24 +define amdgpu_kernel void @extra_and(i32 addrspace(1)* %arg, i32 %arg2, i32 %arg3) { +bb: + br label %bb4 + +bb4: ; preds = %bb4, %bb + %tmp = phi i32 [ 0, %bb ], [ %tmp13, %bb4 ] + %tmp5 = phi i32 [ 0, %bb ], [ %tmp13, %bb4 ] + %tmp6 = phi i32 [ 0, %bb ], [ %tmp15, %bb4 ] + %tmp7 = phi i32 [ 0, %bb ], [ %tmp15, %bb4 ] + %tmp8 = and i32 %tmp7, 16777215 + %tmp9 = and i32 %tmp6, 16777215 + %tmp10 = and i32 %tmp5, 16777215 + %tmp11 = and i32 %tmp, 16777215 + %tmp12 = mul i32 %tmp8, %tmp11 + %tmp13 = add i32 %arg2, %tmp12 + %tmp14 = mul i32 %tmp9, %tmp11 + %tmp15 = add i32 %arg3, %tmp14 + %tmp16 = add nuw nsw i32 %tmp13, %tmp15 + %tmp17 = icmp eq i32 %tmp16, 8 + br i1 %tmp17, label %bb18, label %bb4 + +bb18: ; preds = %bb4 + store i32 %tmp16, i32 addrspace(1)* %arg + ret void +} + +; FUNC-LABEL: {{^}}dont_remove_shift +; SI: v_lshr +; SI: v_mad_u32_u24 +; SI: v_mad_u32_u24 +define amdgpu_kernel void @dont_remove_shift(i32 addrspace(1)* %arg, i32 %arg2, i32 %arg3) { +bb: + br label %bb4 + +bb4: ; preds = %bb4, %bb + %tmp = phi i32 [ 0, %bb ], [ %tmp13, %bb4 ] + %tmp5 = phi i32 [ 0, %bb ], [ %tmp13, %bb4 ] + %tmp6 = phi i32 [ 0, %bb ], [ %tmp15, %bb4 ] + %tmp7 = phi i32 [ 0, %bb ], [ %tmp15, %bb4 ] + %tmp8 = lshr i32 %tmp7, 8 + %tmp9 = lshr i32 %tmp6, 8 + %tmp10 = lshr i32 %tmp5, 8 + %tmp11 = lshr i32 %tmp, 8 + %tmp12 = mul i32 %tmp8, %tmp11 + %tmp13 = add i32 %arg2, %tmp12 + %tmp14 = mul i32 %tmp9, %tmp11 + %tmp15 = add i32 %arg3, %tmp14 + %tmp16 = add nuw nsw i32 %tmp13, %tmp15 + %tmp17 = icmp eq i32 %tmp16, 8 + br i1 %tmp17, label %bb18, label %bb4 + +bb18: ; preds = %bb4 + store i32 %tmp16, i32 addrspace(1)* %arg + ret void +} diff --git a/test/CodeGen/AMDGPU/madak.ll b/test/CodeGen/AMDGPU/madak.ll index 6ea1202ac500..c3d1a49c493a 100644 --- a/test/CodeGen/AMDGPU/madak.ll +++ b/test/CodeGen/AMDGPU/madak.ll @@ -151,7 +151,7 @@ define void @s_s_madak_f32(float addrspace(1)* %out, float %a, float %b) nounwin ; GCN-LABEL: {{^}}no_madak_src0_modifier_f32: ; GCN: buffer_load_dword [[VA:v[0-9]+]] ; GCN: buffer_load_dword [[VB:v[0-9]+]] -; GCN: v_mad_f32 {{v[0-9]+}}, |{{v[0-9]+}}|, {{v[0-9]+}}, {{[sv][0-9]+}} +; GCN: v_mad_f32 {{v[0-9]+}}, {{v[0-9]+}}, |{{v[0-9]+}}|, {{[sv][0-9]+}} ; GCN: s_endpgm define void @no_madak_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind { %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone @@ -173,7 +173,7 @@ define void @no_madak_src0_modifier_f32(float addrspace(1)* noalias %out, float ; GCN-LABEL: {{^}}no_madak_src1_modifier_f32: ; GCN: buffer_load_dword [[VA:v[0-9]+]] ; GCN: buffer_load_dword [[VB:v[0-9]+]] -; GCN: v_mad_f32 {{v[0-9]+}}, {{v[0-9]+}}, |{{v[0-9]+}}|, {{[sv][0-9]+}} +; GCN: v_mad_f32 {{v[0-9]+}}, |{{v[0-9]+}}|, {{v[0-9]+}}, {{[sv][0-9]+}} ; GCN: s_endpgm define void @no_madak_src1_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in.a, float addrspace(1)* noalias %in.b) nounwind { %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone diff --git a/test/CodeGen/AMDGPU/madmk.ll b/test/CodeGen/AMDGPU/madmk.ll index 1adf82402b72..718508526bb1 100644 --- a/test/CodeGen/AMDGPU/madmk.ll +++ b/test/CodeGen/AMDGPU/madmk.ll @@ -128,7 +128,8 @@ define void @scalar_vector_madmk_f32(float addrspace(1)* noalias %out, float add ; GCN-LABEL: {{^}}no_madmk_src0_modifier_f32: ; GCN-DAG: buffer_load_dword [[VA:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} ; GCN-DAG: buffer_load_dword [[VB:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 -; GCN: v_mad_f32 {{v[0-9]+}}, |{{v[0-9]+}}|, {{v[0-9]+}}, {{[sv][0-9]+}} +; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x41200000 +; GCN: v_mad_f32 {{v[0-9]+}}, [[VK]], |[[VA]]|, [[VB]] define void @no_madmk_src0_modifier_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind { %tid = tail call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone %gep.0 = getelementptr float, float addrspace(1)* %in, i32 %tid diff --git a/test/CodeGen/AMDGPU/max.i16.ll b/test/CodeGen/AMDGPU/max.i16.ll new file mode 100644 index 000000000000..0b0e026c5fa1 --- /dev/null +++ b/test/CodeGen/AMDGPU/max.i16.ll @@ -0,0 +1,87 @@ +; RUN: llc < %s -march=amdgcn -mcpu=fiji -verify-machineinstrs | FileCheck -check-prefix=GCN -check-prefix=VI %s + + +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_imax_sge_i16: +; VI: v_max_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_test_imax_sge_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %aptr, i16 addrspace(1)* %bptr) nounwind { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep0 = getelementptr i16, i16 addrspace(1)* %aptr, i32 %tid + %gep1 = getelementptr i16, i16 addrspace(1)* %bptr, i32 %tid + %outgep = getelementptr i16, i16 addrspace(1)* %out, i32 %tid + %a = load i16, i16 addrspace(1)* %gep0, align 4 + %b = load i16, i16 addrspace(1)* %gep1, align 4 + %cmp = icmp sge i16 %a, %b + %val = select i1 %cmp, i16 %a, i16 %b + store i16 %val, i16 addrspace(1)* %outgep, align 4 + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_imax_sge_v4i16: +; VI: v_max_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_max_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_max_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_max_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_test_imax_sge_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %aptr, <4 x i16> addrspace(1)* %bptr) nounwind { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep0 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %aptr, i32 %tid + %gep1 = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %bptr, i32 %tid + %outgep = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %out, i32 %tid + %a = load <4 x i16>, <4 x i16> addrspace(1)* %gep0, align 4 + %b = load <4 x i16>, <4 x i16> addrspace(1)* %gep1, align 4 + %cmp = icmp sge <4 x i16> %a, %b + %val = select <4 x i1> %cmp, <4 x i16> %a, <4 x i16> %b + store <4 x i16> %val, <4 x i16> addrspace(1)* %outgep, align 4 + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_imax_sgt_i16: +; VI: v_max_i16_e32 +define void @v_test_imax_sgt_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %aptr, i16 addrspace(1)* %bptr) nounwind { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep0 = getelementptr i16, i16 addrspace(1)* %aptr, i32 %tid + %gep1 = getelementptr i16, i16 addrspace(1)* %bptr, i32 %tid + %outgep = getelementptr i16, i16 addrspace(1)* %out, i32 %tid + %a = load i16, i16 addrspace(1)* %gep0, align 4 + %b = load i16, i16 addrspace(1)* %gep1, align 4 + %cmp = icmp sgt i16 %a, %b + %val = select i1 %cmp, i16 %a, i16 %b + store i16 %val, i16 addrspace(1)* %outgep, align 4 + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_umax_uge_i16: +; VI: v_max_u16_e32 +define void @v_test_umax_uge_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %aptr, i16 addrspace(1)* %bptr) nounwind { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep0 = getelementptr i16, i16 addrspace(1)* %aptr, i32 %tid + %gep1 = getelementptr i16, i16 addrspace(1)* %bptr, i32 %tid + %outgep = getelementptr i16, i16 addrspace(1)* %out, i32 %tid + %a = load i16, i16 addrspace(1)* %gep0, align 4 + %b = load i16, i16 addrspace(1)* %gep1, align 4 + %cmp = icmp uge i16 %a, %b + %val = select i1 %cmp, i16 %a, i16 %b + store i16 %val, i16 addrspace(1)* %outgep, align 4 + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_umax_ugt_i16: +; VI: v_max_u16_e32 +define void @v_test_umax_ugt_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %aptr, i16 addrspace(1)* %bptr) nounwind { + %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + %gep0 = getelementptr i16, i16 addrspace(1)* %aptr, i32 %tid + %gep1 = getelementptr i16, i16 addrspace(1)* %bptr, i32 %tid + %outgep = getelementptr i16, i16 addrspace(1)* %out, i32 %tid + %a = load i16, i16 addrspace(1)* %gep0, align 4 + %b = load i16, i16 addrspace(1)* %gep1, align 4 + %cmp = icmp ugt i16 %a, %b + %val = select i1 %cmp, i16 %a, i16 %b + store i16 %val, i16 addrspace(1)* %outgep, align 4 + ret void +} diff --git a/test/CodeGen/AMDGPU/mem-builtins.ll b/test/CodeGen/AMDGPU/mem-builtins.ll new file mode 100644 index 000000000000..97512670f59e --- /dev/null +++ b/test/CodeGen/AMDGPU/mem-builtins.ll @@ -0,0 +1,54 @@ +; RUN: not llc -march=amdgcn -verify-machineinstrs < %s 2>&1 | FileCheck -check-prefix=ERROR %s + +declare i32 @memcmp(i8 addrspace(1)* readonly nocapture, i8 addrspace(1)* readonly nocapture, i64) #0 +declare i8 addrspace(1)* @memchr(i8 addrspace(1)* readonly nocapture, i32, i64) #1 +declare i8* @strcpy(i8* nocapture, i8* readonly nocapture) #0 +declare i32 @strlen(i8* nocapture) #1 +declare i32 @strnlen(i8* nocapture, i32) #1 +declare i32 @strcmp(i8* nocapture, i8* nocapture) #1 + + +; ERROR: error: <unknown>:0:0: in function test_memcmp void (i8 addrspace(1)*, i8 addrspace(1)*, i32*): unsupported call to function memcmp +define void @test_memcmp(i8 addrspace(1)* %x, i8 addrspace(1)* %y, i32* nocapture %p) #0 { +entry: + %cmp = tail call i32 @memcmp(i8 addrspace(1)* %x, i8 addrspace(1)* %y, i64 2) + store volatile i32 %cmp, i32 addrspace(1)* undef + ret void +} + +; ERROR: error: <unknown>:0:0: in function test_memchr void (i8 addrspace(1)*, i32, i64): unsupported call to function memchr +define void @test_memchr(i8 addrspace(1)* %src, i32 %char, i64 %len) #0 { + %res = call i8 addrspace(1)* @memchr(i8 addrspace(1)* %src, i32 %char, i64 %len) + store volatile i8 addrspace(1)* %res, i8 addrspace(1)* addrspace(1)* undef + ret void +} + +; ERROR: error: <unknown>:0:0: in function test_strcpy void (i8*, i8*): unsupported call to function strcpy +define void @test_strcpy(i8* %dst, i8* %src) #0 { + %res = call i8* @strcpy(i8* %dst, i8* %src) + store volatile i8* %res, i8* addrspace(1)* undef + ret void +} + +; ERROR: error: <unknown>:0:0: in function test_strcmp void (i8*, i8*): unsupported call to function strcmp +define void @test_strcmp(i8* %src0, i8* %src1) #0 { + %res = call i32 @strcmp(i8* %src0, i8* %src1) + store volatile i32 %res, i32 addrspace(1)* undef + ret void +} + +; ERROR: error: <unknown>:0:0: in function test_strlen void (i8*): unsupported call to function strlen +define void @test_strlen(i8* %src) #0 { + %res = call i32 @strlen(i8* %src) + store volatile i32 %res, i32 addrspace(1)* undef + ret void +} + +; ERROR: error: <unknown>:0:0: in function test_strnlen void (i8*, i32): unsupported call to function strnlen +define void @test_strnlen(i8* %src, i32 %size) #0 { + %res = call i32 @strnlen(i8* %src, i32 %size) + store volatile i32 %res, i32 addrspace(1)* undef + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/merge-store-crash.ll b/test/CodeGen/AMDGPU/merge-store-crash.ll new file mode 100644 index 000000000000..ef552e295fd4 --- /dev/null +++ b/test/CodeGen/AMDGPU/merge-store-crash.ll @@ -0,0 +1,36 @@ +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s + +; This is used to crash in LiveIntervalAnalysis via SILoadStoreOptimizer +; while fixing up the merge of two ds_write instructions. + +@tess_lds = external addrspace(3) global [8192 x i32] + +; CHECK-LABEL: {{^}}main: +; CHECK: ds_write2_b32 +; CHECK: v_mov_b32_e32 v1, v0 +; CHECK: tbuffer_store_format_xyzw v[0:3], +define amdgpu_vs void @main(i32 inreg %arg) { +main_body: + %tmp = load float, float addrspace(3)* undef, align 4 + %tmp1 = load float, float addrspace(3)* undef, align 4 + store float %tmp, float addrspace(3)* null, align 4 + %tmp2 = bitcast float %tmp to i32 + %tmp3 = add nuw nsw i32 0, 1 + %tmp4 = zext i32 %tmp3 to i64 + %tmp5 = getelementptr [8192 x i32], [8192 x i32] addrspace(3)* @tess_lds, i64 0, i64 %tmp4 + %tmp6 = bitcast i32 addrspace(3)* %tmp5 to float addrspace(3)* + store float %tmp1, float addrspace(3)* %tmp6, align 4 + %tmp7 = bitcast float %tmp1 to i32 + %tmp8 = insertelement <4 x i32> undef, i32 %tmp2, i32 0 + %tmp9 = insertelement <4 x i32> %tmp8, i32 %tmp7, i32 1 + %tmp10 = insertelement <4 x i32> %tmp9, i32 undef, i32 2 + %tmp11 = insertelement <4 x i32> %tmp10, i32 undef, i32 3 + call void @llvm.SI.tbuffer.store.v4i32(<16 x i8> undef, <4 x i32> %tmp11, i32 4, i32 undef, i32 %arg, i32 0, i32 14, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + ret void +} + +; Function Attrs: nounwind +declare void @llvm.SI.tbuffer.store.v4i32(<16 x i8>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) #0 + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/merge-store-usedef.ll b/test/CodeGen/AMDGPU/merge-store-usedef.ll new file mode 100644 index 000000000000..e4a36d7e6914 --- /dev/null +++ b/test/CodeGen/AMDGPU/merge-store-usedef.ll @@ -0,0 +1,23 @@ +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: {{^}}test1: +; CHECK: ds_write_b32 +; CHECK: ds_read_b32 +; CHECK: ds_write_b32 +define amdgpu_vs void @test1(i32 %v) #0 { + %p0 = getelementptr i32, i32 addrspace(3)* null, i32 0 + %p1 = getelementptr i32, i32 addrspace(3)* null, i32 1 + + store i32 %v, i32 addrspace(3)* %p0 + + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 %v, i32 1, i32 undef, i32 undef, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + + %w = load i32, i32 addrspace(3)* %p0 + store i32 %w, i32 addrspace(3)* %p1 + ret void +} + +declare void @llvm.SI.tbuffer.store.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) #0 + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/merge-stores.ll b/test/CodeGen/AMDGPU/merge-stores.ll index 17b4af818f8f..07104ebc8c97 100644 --- a/test/CodeGen/AMDGPU/merge-stores.ll +++ b/test/CodeGen/AMDGPU/merge-stores.ll @@ -1,9 +1,10 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-NOAA %s -; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-NOAA %s +; RUN: llc -march=amdgcn -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-NOAA %s +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-NOAA %s -; RUN: llc -march=amdgcn -verify-machineinstrs -combiner-alias-analysis < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s -; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -combiner-alias-analysis < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s +; RUN: llc -march=amdgcn -verify-machineinstrs -combiner-alias-analysis -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -combiner-alias-analysis -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s +; This test is mostly to test DAG store merging, so disable the vectorizer. ; Run with devices with different unaligned load restrictions. ; TODO: Vector element tests @@ -148,12 +149,8 @@ define void @merge_global_store_4_constants_f32(float addrspace(1)* %out) #0 { ret void } -; FIXME: Should be able to merge this ; GCN-LABEL: {{^}}merge_global_store_4_constants_mixed_i32_f32: -; GCN-NOAA: buffer_store_dword v -; GCN-NOAA: buffer_store_dword v -; GCN-NOAA: buffer_store_dword v -; GCN-NOAA: buffer_store_dword v +; GCN-NOAA: buffer_store_dwordx4 v ; GCN-AA: buffer_store_dwordx2 ; GCN-AA: buffer_store_dword v diff --git a/test/CodeGen/AMDGPU/mesa_regression.ll b/test/CodeGen/AMDGPU/mesa_regression.ll new file mode 100644 index 000000000000..90c4fe2ff54a --- /dev/null +++ b/test/CodeGen/AMDGPU/mesa_regression.ll @@ -0,0 +1,11 @@ +; RUN: llc -O2 -mtriple amdgcn--amdhsa -mcpu=fiji -amdgpu-scalarize-global-loads=false -verify-machineinstrs < %s | FileCheck %s + +; CHECK-LABEL: %entry +; CHECK: flat_load_dwordx4 + +define amdgpu_kernel void @store_global(<16 x double> addrspace(1)* nocapture %out, <16 x double> addrspace(1)* nocapture readonly %in) { +entry: + %tmp = load <16 x double>, <16 x double> addrspace(1)* %in + store <16 x double> %tmp, <16 x double> addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/missing-store.ll b/test/CodeGen/AMDGPU/missing-store.ll index 3d6d7fae0fd6..8e1b0036a1af 100644 --- a/test/CodeGen/AMDGPU/missing-store.ll +++ b/test/CodeGen/AMDGPU/missing-store.ll @@ -6,12 +6,12 @@ ; resulting in losing the store to gptr ; FUNC-LABEL: {{^}}missing_store_reduced: +; SI: s_load_dwordx2 ; SI: ds_read_b64 ; SI-DAG: buffer_store_dword ; SI-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}} ; SI: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}} -; SI: s_load_dword -; SI: s_nop 2 +; SI: s_nop 3 ; SI: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}} ; SI: buffer_store_dword ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll b/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll index 36f12573c173..85dfbe6b8a33 100644 --- a/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll +++ b/test/CodeGen/AMDGPU/move-addr64-rsrc-dead-subreg-writes.ll @@ -10,10 +10,10 @@ ; GCN-DAG: buffer_load_dwordx2 v{{\[}}[[LDPTRLO:[0-9]+]]:[[LDPTRHI:[0-9]+]]{{\]}} ; GCN-NOT: v_mov_b32 -; GCN: v_mov_b32_e32 v[[VARG1HI:[0-9]+]], s[[ARG1HI]] -; GCN-NOT: v_mov_b32 ; GCN: v_mov_b32_e32 v[[VARG1LO:[0-9]+]], s[[ARG1LO]] ; GCN-NOT: v_mov_b32 +; GCN: v_mov_b32_e32 v[[VARG1HI:[0-9]+]], s[[ARG1HI]] +; GCN-NOT: v_mov_b32 ; GCN: v_add_i32_e32 v[[PTRLO:[0-9]+]], vcc, v[[LDPTRLO]], v[[VARG1LO]] ; GCN: v_addc_u32_e32 v[[PTRHI:[0-9]+]], vcc, v[[LDPTRHI]], v[[VARG1HI]] diff --git a/test/CodeGen/AMDGPU/movreld-bug.ll b/test/CodeGen/AMDGPU/movreld-bug.ll new file mode 100644 index 000000000000..0cc8a5271bd7 --- /dev/null +++ b/test/CodeGen/AMDGPU/movreld-bug.ll @@ -0,0 +1,15 @@ +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}main: +; GCN: v_movreld_b32_e32 v0, +; GCN: v_mov_b32_e32 v0, v1 +; GCN: ; return +define amdgpu_ps float @main(i32 inreg %arg) #0 { +main_body: + %tmp24 = insertelement <2 x float> undef, float 0.000000e+00, i32 %arg + %tmp25 = extractelement <2 x float> %tmp24, i32 1 + ret float %tmp25 +} + +attributes #0 = { "InitialPSInputAddr"="36983" } diff --git a/test/CodeGen/AMDGPU/movrels-bug.mir b/test/CodeGen/AMDGPU/movrels-bug.mir new file mode 100644 index 000000000000..9c330bc8a6be --- /dev/null +++ b/test/CodeGen/AMDGPU/movrels-bug.mir @@ -0,0 +1,31 @@ +# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass post-RA-sched %s -o - | FileCheck %s + +# This tests a situation where a sub-register of a killed super-register operand +# of V_MOVRELS happens to have an undef use later on. This leads to the post RA +# scheduler adding additional implicit operands to the V_MOVRELS, which used +# to fail machine instruction verification. + +--- | + + define amdgpu_vs void @main(i32 %arg) { ret void } + +... +--- +# CHECK-LABEL: name: main +# CHECK-LABEL: bb.0: +# CHECK: V_MOVRELS_B32_e32 +# CHECK: V_MAC_F32_e32 + +name: main +tracksRegLiveness: true +body: | + bb.0: + %m0 = S_MOV_B32 undef %sgpr0 + V_MOVRELD_B32_e32 undef %vgpr2, 0, implicit %m0, implicit %exec, implicit-def %vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8, implicit undef %vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8(tied-def 4) + %m0 = S_MOV_B32 undef %sgpr0 + %vgpr1 = V_MOVRELS_B32_e32 undef %vgpr1, implicit %m0, implicit %exec, implicit killed %vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 + %vgpr4 = V_MAC_F32_e32 undef %vgpr0, undef %vgpr0, undef %vgpr4, implicit %exec + EXP_DONE 15, undef %vgpr0, killed %vgpr1, killed %vgpr4, undef %vgpr0, 0, 0, 12, implicit %exec + S_ENDPGM + +... diff --git a/test/CodeGen/AMDGPU/mul.ll b/test/CodeGen/AMDGPU/mul.ll index 5ceef7fda826..c2fae4cfd2b2 100644 --- a/test/CodeGen/AMDGPU/mul.ll +++ b/test/CodeGen/AMDGPU/mul.ll @@ -198,3 +198,74 @@ endif: store i64 %3, i64 addrspace(1)* %out ret void } + +; FIXME: Load dwordx4 +; FUNC-LABEL: {{^}}s_mul_i128: +; SI: s_load_dwordx2 +; SI: s_load_dwordx2 +; SI: s_load_dwordx2 +; SI: s_load_dwordx2 + +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: s_mul_i32 +; SI: v_mul_hi_u32 +; SI: s_mul_i32 +; SI: s_mul_i32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: s_mul_i32 +; SI-DAG: s_mul_i32 +; SI-DAG: v_mul_hi_u32 +; SI: s_mul_i32 +; SI: s_mul_i32 +; SI: s_mul_i32 +; SI: s_mul_i32 +; SI: s_mul_i32 + +; SI: buffer_store_dwordx4 +define void @s_mul_i128(i128 addrspace(1)* %out, i128 %a, i128 %b) nounwind #0 { + %mul = mul i128 %a, %b + store i128 %mul, i128 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_mul_i128: +; SI: {{buffer|flat}}_load_dwordx4 +; SI: {{buffer|flat}}_load_dwordx4 + +; SI: v_mul_lo_i32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: v_mul_lo_i32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: v_mul_lo_i32 +; SI: v_mul_lo_i32 +; SI: v_add_i32_e32 +; SI: v_mul_hi_u32 +; SI: v_mul_lo_i32 +; SI: v_mul_hi_u32 +; SI: v_mul_lo_i32 +; SI: v_mul_lo_i32 +; SI: v_mul_lo_i32 +; SI: v_mul_lo_i32 +; SI: v_mul_lo_i32 + +; SI: {{buffer|flat}}_store_dwordx4 +define void @v_mul_i128(i128 addrspace(1)* %out, i128 addrspace(1)* %aptr, i128 addrspace(1)* %bptr) #0 { + %tid = call i32 @llvm.r600.read.tidig.x() + %gep.a = getelementptr inbounds i128, i128 addrspace(1)* %aptr, i32 %tid + %gep.b = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid + %gep.out = getelementptr inbounds i128, i128 addrspace(1)* %bptr, i32 %tid + %a = load i128, i128 addrspace(1)* %gep.a + %b = load i128, i128 addrspace(1)* %gep.b + %mul = mul i128 %a, %b + store i128 %mul, i128 addrspace(1)* %gep.out + ret void +} + +declare i32 @llvm.r600.read.tidig.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone} diff --git a/test/CodeGen/AMDGPU/mul_int24.ll b/test/CodeGen/AMDGPU/mul_int24.ll index 1a323fbaa1a3..0beb9524ed79 100644 --- a/test/CodeGen/AMDGPU/mul_int24.ll +++ b/test/CodeGen/AMDGPU/mul_int24.ll @@ -1,23 +1,174 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=CM --check-prefix=FUNC +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s + +; FUNC-LABEL: {{^}}test_smul24_i32: +; GCN-NOT: bfe +; GCN: v_mul_i32_i24 -; FUNC-LABEL: {{^}}i32_mul24: ; Signed 24-bit multiply is not supported on pre-Cayman GPUs. ; EG: MULLO_INT + ; Make sure we are not masking the inputs ; CM-NOT: AND ; CM: MUL_INT24 -; SI-NOT: and -; SI: v_mul_i32_i24 -define void @i32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) { +define void @test_smul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { +entry: + %a.shl = shl i32 %a, 8 + %a.24 = ashr i32 %a.shl, 8 + %b.shl = shl i32 %b, 8 + %b.24 = ashr i32 %b.shl, 8 + %mul24 = mul i32 %a.24, %b.24 + store i32 %mul24, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smulhi24_i64: +; GCN-NOT: bfe +; GCN-NOT: ashr +; GCN: v_mul_hi_i32_i24_e32 [[RESULT:v[0-9]+]], +; GCN-NEXT: buffer_store_dword [[RESULT]] + +; EG: ASHR +; EG: ASHR +; EG: MULHI_INT + +; CM-NOT: ASHR +; CM: MULHI_INT24 +; CM: MULHI_INT24 +; CM: MULHI_INT24 +; CM: MULHI_INT24 +define void @test_smulhi24_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: - %0 = shl i32 %a, 8 - %a_24 = ashr i32 %0, 8 - %1 = shl i32 %b, 8 - %b_24 = ashr i32 %1, 8 - %2 = mul i32 %a_24, %b_24 - store i32 %2, i32 addrspace(1)* %out + %a.shl = shl i32 %a, 8 + %a.24 = ashr i32 %a.shl, 8 + %b.shl = shl i32 %b, 8 + %b.24 = ashr i32 %b.shl, 8 + %a.24.i64 = sext i32 %a.24 to i64 + %b.24.i64 = sext i32 %b.24 to i64 + %mul48 = mul i64 %a.24.i64, %b.24.i64 + %mul48.hi = lshr i64 %mul48, 32 + %mul24hi = trunc i64 %mul48.hi to i32 + store i32 %mul24hi, i32 addrspace(1)* %out + ret void +} + +; This requires handling of the original 64-bit mul node to eliminate +; unnecessary extension instructions because after legalization they +; will not be removed by SimplifyDemandedBits because there are +; multiple uses by the separate mul and mulhi. + +; FUNC-LABEL: {{^}}test_smul24_i64: +; GCN: s_load_dword s +; GCN: s_load_dword s + +; GCN-NOT: bfe +; GCN-NOT: ashr + +; GCN-DAG: v_mul_hi_i32_i24_e32 +; GCN-DAG: v_mul_i32_i24_e32 + +; GCN: buffer_store_dwordx2 +define void @test_smul24_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 { + %shl.i = shl i32 %a, 8 + %shr.i = ashr i32 %shl.i, 8 + %conv.i = sext i32 %shr.i to i64 + %shl1.i = shl i32 %b, 8 + %shr2.i = ashr i32 %shl1.i, 8 + %conv3.i = sext i32 %shr2.i to i64 + %mul.i = mul i64 %conv3.i, %conv.i + store i64 %mul.i, i64 addrspace(1)* %out ret void } + +; FUNC-LABEL: {{^}}test_smul24_i64_square: +; GCN: s_load_dword [[A:s[0-9]+]] +; GCN-DAG: v_mul_hi_i32_i24_e64 v{{[0-9]+}}, [[A]], [[A]] +; GCN-DAG: v_mul_i32_i24_e64 v{{[0-9]+}}, [[A]], [[A]] +; GCN: buffer_store_dwordx2 +define void @test_smul24_i64_square(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 { + %shl.i = shl i32 %a, 8 + %shr.i = ashr i32 %shl.i, 8 + %conv.i = sext i32 %shr.i to i64 + %mul.i = mul i64 %conv.i, %conv.i + store i64 %mul.i, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smul24_i33: +; GCN: s_load_dword s +; GCN: s_load_dword s + +; GCN-NOT: and +; GCN-NOT: lshr + +; GCN-DAG: v_mul_i32_i24_e32 +; GCN-DAG: v_mul_hi_i32_i24_e32 +; SI: v_lshl_b64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31 +; SI: v_ashr_i64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 31 + +; VI: v_lshlrev_b64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}} +; VI: v_ashrrev_i64 v{{\[[0-9]+:[0-9]+\]}}, 31, v{{\[[0-9]+:[0-9]+\]}} + +; GCN: buffer_store_dwordx2 +define void @test_smul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) #0 { +entry: + %a.shl = shl i33 %a, 9 + %a.24 = ashr i33 %a.shl, 9 + %b.shl = shl i33 %b, 9 + %b.24 = ashr i33 %b.shl, 9 + %mul24 = mul i33 %a.24, %b.24 + %ext = sext i33 %mul24 to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_smulhi24_i33: +; SI: s_load_dword s +; SI: s_load_dword s + +; SI-NOT: bfe + +; SI: v_mul_hi_i32_i24_e32 v[[MUL_HI:[0-9]+]], +; SI-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; SI-NEXT: buffer_store_dword v[[HI]] +define void @test_smulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = ashr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = ashr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %hi = lshr i33 %tmp2, 32 + %trunc = trunc i33 %hi to i32 + + store i32 %trunc, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}simplify_i24_crash: +; GCN: v_mul_i32_i24_e32 v[[VAL_LO:[0-9]+]] +; GCN: v_mov_b32_e32 v[[VAL_HI:[0-9]+]], v[[VAL_LO]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VAL_LO]]:[[VAL_HI]]{{\]}} +define void @simplify_i24_crash(<2 x i32> addrspace(1)* %out, i32 %arg0, <2 x i32> %arg1, <2 x i32> %arg2) { +bb: + %cmp = icmp eq i32 %arg0, 0 + br i1 %cmp, label %bb11, label %bb7 + +bb11: + %tmp14 = shufflevector <2 x i32> %arg1, <2 x i32> undef, <2 x i32> zeroinitializer + %tmp16 = shufflevector <2 x i32> %arg2, <2 x i32> undef, <2 x i32> zeroinitializer + %tmp17 = shl <2 x i32> %tmp14, <i32 8, i32 8> + %tmp18 = ashr <2 x i32> %tmp17, <i32 8, i32 8> + %tmp19 = shl <2 x i32> %tmp16, <i32 8, i32 8> + %tmp20 = ashr <2 x i32> %tmp19, <i32 8, i32 8> + %tmp21 = mul <2 x i32> %tmp18, %tmp20 + store <2 x i32> %tmp21, <2 x i32> addrspace(1)* %out + br label %bb7 + +bb7: + ret void + +} +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll b/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll new file mode 100644 index 000000000000..76869337730a --- /dev/null +++ b/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll @@ -0,0 +1,220 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s + +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone +declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone + +; FUNC-LABEL: {{^}}test_umul24_i32: +; GCN: v_mul_u32_u24 +define void @test_umul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %0 = shl i32 %a, 8 + %a_24 = lshr i32 %0, 8 + %1 = shl i32 %b, 8 + %b_24 = lshr i32 %1, 8 + %2 = mul i32 %a_24, %b_24 + store i32 %2, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i16_sext: +; SI: v_mul_u32_u24_e{{(32|64)}} [[VI_MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; SI: v_bfe_i32 v{{[0-9]}}, [[VI_MUL]], 0, 16 +; VI: s_mul_i32 [[SI_MUL:s[0-9]]], s{{[0-9]}}, s{{[0-9]}} +; VI: s_sext_i32_i16 s{{[0-9]}}, [[SI_MUL]] +define void @test_umul24_i16_sext(i32 addrspace(1)* %out, i16 %a, i16 %b) { +entry: + %mul = mul i16 %a, %b + %ext = sext i16 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i16_vgpr_sext: +; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; VI: v_mul_lo_u16_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; GCN: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 16 +define void @test_umul24_i16_vgpr_sext(i32 addrspace(1)* %out, i16 addrspace(1)* %in) { + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %tid.y = call i32 @llvm.amdgcn.workitem.id.y() + %ptr_a = getelementptr i16, i16 addrspace(1)* %in, i32 %tid.x + %ptr_b = getelementptr i16, i16 addrspace(1)* %in, i32 %tid.y + %a = load i16, i16 addrspace(1)* %ptr_a + %b = load i16, i16 addrspace(1)* %ptr_b + %mul = mul i16 %a, %b + %val = sext i16 %mul to i32 + store i32 %val, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i16: +; SI: s_and_b32 +; SI: v_mul_u32_u24_e32 +; SI: v_and_b32_e32 +; VI: s_mul_i32 +; VI: s_and_b32 +; VI: v_mov_b32_e32 +define void @test_umul24_i16(i32 addrspace(1)* %out, i16 %a, i16 %b) { +entry: + %mul = mul i16 %a, %b + %ext = zext i16 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i16_vgpr: +; SI: v_mul_u32_u24_e32 +; SI: v_and_b32_e32 +; VI: v_mul_lo_u16 +define void @test_umul24_i16_vgpr(i32 addrspace(1)* %out, i16 addrspace(1)* %in) { + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %tid.y = call i32 @llvm.amdgcn.workitem.id.y() + %ptr_a = getelementptr i16, i16 addrspace(1)* %in, i32 %tid.x + %ptr_b = getelementptr i16, i16 addrspace(1)* %in, i32 %tid.y + %a = load i16, i16 addrspace(1)* %ptr_a + %b = load i16, i16 addrspace(1)* %ptr_b + %mul = mul i16 %a, %b + %val = zext i16 %mul to i32 + store i32 %val, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i8_vgpr: +; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; VI: v_mul_lo_u16_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} +; GCN: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8 +define void @test_umul24_i8_vgpr(i32 addrspace(1)* %out, i8 addrspace(1)* %a, i8 addrspace(1)* %b) { +entry: + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %tid.y = call i32 @llvm.amdgcn.workitem.id.y() + %a.ptr = getelementptr i8, i8 addrspace(1)* %a, i32 %tid.x + %b.ptr = getelementptr i8, i8 addrspace(1)* %b, i32 %tid.y + %a.l = load i8, i8 addrspace(1)* %a.ptr + %b.l = load i8, i8 addrspace(1)* %b.ptr + %mul = mul i8 %a.l, %b.l + %ext = sext i8 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24_i32_i64: +; GCN-NOT: and +; GCN: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]], +; GCN-NEXT: buffer_store_dword [[RESULT]] +define void @test_umulhi24_i32_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %a.24 = and i32 %a, 16777215 + %b.24 = and i32 %b, 16777215 + %a.24.i64 = zext i32 %a.24 to i64 + %b.24.i64 = zext i32 %b.24 to i64 + %mul48 = mul i64 %a.24.i64, %b.24.i64 + %mul48.hi = lshr i64 %mul48, 32 + %mul24hi = trunc i64 %mul48.hi to i32 + store i32 %mul24hi, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24: +; GCN-NOT: and +; GCN: v_mul_hi_u32_u24_e32 [[RESULT:v[0-9]+]], +; GCN-NEXT: buffer_store_dword [[RESULT]] +define void @test_umulhi24(i32 addrspace(1)* %out, i64 %a, i64 %b) { +entry: + %a.24 = and i64 %a, 16777215 + %b.24 = and i64 %b, 16777215 + %mul48 = mul i64 %a.24, %b.24 + %mul48.hi = lshr i64 %mul48, 32 + %mul24.hi = trunc i64 %mul48.hi to i32 + store i32 %mul24.hi, i32 addrspace(1)* %out + ret void +} + +; Multiply with 24-bit inputs and 64-bit output. +; FUNC-LABEL: {{^}}test_umul24_i64: +; GCN-NOT: and +; GCN-NOT: lshr +; GCN-DAG: v_mul_u32_u24_e32 +; GCN-DAG: v_mul_hi_u32_u24_e32 +; GCN: buffer_store_dwordx2 +define void @test_umul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { +entry: + %tmp0 = shl i64 %a, 40 + %a_24 = lshr i64 %tmp0, 40 + %tmp1 = shl i64 %b, 40 + %b_24 = lshr i64 %tmp1, 40 + %tmp2 = mul i64 %a_24, %b_24 + store i64 %tmp2, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i64_square: +; GCN: s_load_dword [[A:s[0-9]+]] +; GCN-NOT: s_and_b32 +; GCN-DAG: v_mul_hi_u32_u24_e64 v{{[0-9]+}}, [[A]], [[A]] +; GCN-DAG: v_mul_u32_u24_e64 v{{[0-9]+}}, [[A]], [[A]] +define void @test_umul24_i64_square(i64 addrspace(1)* %out, i64 %a) { +entry: + %tmp0 = shl i64 %a, 40 + %a.24 = lshr i64 %tmp0, 40 + %tmp2 = mul i64 %a.24, %a.24 + store i64 %tmp2, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi16_i32: +; GCN: s_and_b32 +; GCN: s_and_b32 +; GCN: v_mul_u32_u24_e32 [[MUL24:v[0-9]+]] +; GCN: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, [[MUL24]] +define void @test_umulhi16_i32(i16 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %a.16 = and i32 %a, 65535 + %b.16 = and i32 %b, 65535 + %mul = mul i32 %a.16, %b.16 + %hi = lshr i32 %mul, 16 + %mulhi = trunc i32 %hi to i16 + store i16 %mulhi, i16 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umul24_i33: +; GCN: s_load_dword s +; GCN: s_load_dword s +; GCN-NOT: and +; GCN-NOT: lshr +; GCN-DAG: v_mul_u32_u24_e32 v[[MUL_LO:[0-9]+]], +; GCN-DAG: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]], +; GCN-DAG: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[MUL_LO]]:[[HI]]{{\]}} +define void @test_umul24_i33(i64 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = lshr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = lshr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %ext = zext i33 %tmp2 to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24_i33: +; GCN: s_load_dword s +; GCN: s_load_dword s +; GCN-NOT: and +; GCN-NOT: lshr +; GCN: v_mul_hi_u32_u24_e32 v[[MUL_HI:[0-9]+]], +; GCN-NEXT: v_and_b32_e32 v[[HI:[0-9]+]], 1, v[[MUL_HI]] +; GCN-NEXT: buffer_store_dword v[[HI]] +define void @test_umulhi24_i33(i32 addrspace(1)* %out, i33 %a, i33 %b) { +entry: + %tmp0 = shl i33 %a, 9 + %a_24 = lshr i33 %tmp0, 9 + %tmp1 = shl i33 %b, 9 + %b_24 = lshr i33 %tmp1, 9 + %tmp2 = mul i33 %a_24, %b_24 + %hi = lshr i33 %tmp2, 32 + %trunc = trunc i33 %hi to i32 + store i32 %trunc, i32 addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/mul_uint24-r600.ll b/test/CodeGen/AMDGPU/mul_uint24-r600.ll new file mode 100644 index 000000000000..da1c111fa5c0 --- /dev/null +++ b/test/CodeGen/AMDGPU/mul_uint24-r600.ll @@ -0,0 +1,83 @@ +; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s + +; FUNC-LABEL: {{^}}test_umul24_i32: +; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W +define void @test_umul24_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %0 = shl i32 %a, 8 + %a_24 = lshr i32 %0, 8 + %1 = shl i32 %b, 8 + %b_24 = lshr i32 %1, 8 + %2 = mul i32 %a_24, %b_24 + store i32 %2, i32 addrspace(1)* %out + ret void +} + +; The result must be sign-extended. +; FUNC-LABEL: {{^}}test_umul24_i16_sext: +; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] +; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x +; EG: 16 +define void @test_umul24_i16_sext(i32 addrspace(1)* %out, i16 %a, i16 %b) { +entry: + %mul = mul i16 %a, %b + %ext = sext i16 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; The result must be sign-extended. +; FUNC-LABEL: {{^}}test_umul24_i8: +; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] +; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x +define void @test_umul24_i8(i32 addrspace(1)* %out, i8 %a, i8 %b) { +entry: + %mul = mul i8 %a, %b + %ext = sext i8 %mul to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24_i32_i64: +; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W +define void @test_umulhi24_i32_i64(i32 addrspace(1)* %out, i32 %a, i32 %b) { +entry: + %a.24 = and i32 %a, 16777215 + %b.24 = and i32 %b, 16777215 + %a.24.i64 = zext i32 %a.24 to i64 + %b.24.i64 = zext i32 %b.24 to i64 + %mul48 = mul i64 %a.24.i64, %b.24.i64 + %mul48.hi = lshr i64 %mul48, 32 + %mul24hi = trunc i64 %mul48.hi to i32 + store i32 %mul24hi, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_umulhi24: +; EG: MULHI_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].W, KC0[3].Y +define void @test_umulhi24(i32 addrspace(1)* %out, i64 %a, i64 %b) { +entry: + %a.24 = and i64 %a, 16777215 + %b.24 = and i64 %b, 16777215 + %mul48 = mul i64 %a.24, %b.24 + %mul48.hi = lshr i64 %mul48, 32 + %mul24.hi = trunc i64 %mul48.hi to i32 + store i32 %mul24.hi, i32 addrspace(1)* %out + ret void +} + +; Multiply with 24-bit inputs and 64-bit output. +; FUNC-LABEL: {{^}}test_umul24_i64: +; EG; MUL_UINT24 +; EG: MULHI +define void @test_umul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { +entry: + %tmp0 = shl i64 %a, 40 + %a_24 = lshr i64 %tmp0, 40 + %tmp1 = shl i64 %b, 40 + %b_24 = lshr i64 %tmp1, 40 + %tmp2 = mul i64 %a_24, %b_24 + store i64 %tmp2, i64 addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/mul_uint24.ll b/test/CodeGen/AMDGPU/mul_uint24.ll deleted file mode 100644 index fdd348403edf..000000000000 --- a/test/CodeGen/AMDGPU/mul_uint24.ll +++ /dev/null @@ -1,69 +0,0 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=EG --check-prefix=FUNC -; RUN: llc < %s -march=r600 -mcpu=cayman | FileCheck %s --check-prefix=EG --check-prefix=FUNC - -; FUNC-LABEL: {{^}}u32_mul24: -; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]\.[XYZW]}}, KC0[2].Z, KC0[2].W -; SI: v_mul_u32_u24 - -define void @u32_mul24(i32 addrspace(1)* %out, i32 %a, i32 %b) { -entry: - %0 = shl i32 %a, 8 - %a_24 = lshr i32 %0, 8 - %1 = shl i32 %b, 8 - %b_24 = lshr i32 %1, 8 - %2 = mul i32 %a_24, %b_24 - store i32 %2, i32 addrspace(1)* %out - ret void -} - -; FUNC-LABEL: {{^}}i16_mul24: -; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] -; The result must be sign-extended -; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x -; EG: 16 -; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} -; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 16 -define void @i16_mul24(i32 addrspace(1)* %out, i16 %a, i16 %b) { -entry: - %0 = mul i16 %a, %b - %1 = sext i16 %0 to i32 - store i32 %1, i32 addrspace(1)* %out - ret void -} - -; FUNC-LABEL: {{^}}i8_mul24: -; EG: MUL_UINT24 {{[* ]*}}T{{[0-9]}}.[[MUL_CHAN:[XYZW]]] -; The result must be sign-extended -; EG: BFE_INT {{[* ]*}}T{{[0-9]}}.{{[XYZW]}}, PV.[[MUL_CHAN]], 0.0, literal.x -; SI: v_mul_u32_u24_e{{(32|64)}} [[MUL:v[0-9]]], {{[sv][0-9], [sv][0-9]}} -; SI: v_bfe_i32 v{{[0-9]}}, [[MUL]], 0, 8 - -define void @i8_mul24(i32 addrspace(1)* %out, i8 %a, i8 %b) { -entry: - %0 = mul i8 %a, %b - %1 = sext i8 %0 to i32 - store i32 %1, i32 addrspace(1)* %out - ret void -} - -; Multiply with 24-bit inputs and 64-bit output -; FUNC_LABEL: {{^}}mul24_i64: -; EG; MUL_UINT24 -; EG: MULHI -; FIXME: SI support 24-bit mulhi - -; SI-DAG: v_mul_u32_u24 -; SI-DAG: v_mul_hi_u32 -; SI: s_endpgm -define void @mul24_i64(i64 addrspace(1)* %out, i64 %a, i64 %b, i64 %c) { -entry: - %tmp0 = shl i64 %a, 40 - %a_24 = lshr i64 %tmp0, 40 - %tmp1 = shl i64 %b, 40 - %b_24 = lshr i64 %tmp1, 40 - %tmp2 = mul i64 %a_24, %b_24 - store i64 %tmp2, i64 addrspace(1)* %out - ret void -} diff --git a/test/CodeGen/AMDGPU/multilevel-break.ll b/test/CodeGen/AMDGPU/multilevel-break.ll index e1130c9125e5..95c7ce862329 100644 --- a/test/CodeGen/AMDGPU/multilevel-break.ll +++ b/test/CodeGen/AMDGPU/multilevel-break.ll @@ -1,22 +1,48 @@ -; RUN: opt -S -mtriple=amdgcn-- -structurizecfg -si-annotate-control-flow < %s | FileCheck %s +; RUN: opt -S -mtriple=amdgcn-- -structurizecfg -si-annotate-control-flow < %s | FileCheck -check-prefix=OPT %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s -; CHECK-LABEL: {{^}}define amdgpu_vs void @main -; CHECK: main_body: -; CHECK: LOOP.outer: -; CHECK: LOOP: -; CHECK: [[if:%[0-9]+]] = call { i1, i64 } @llvm.amdgcn.if( -; CHECK: [[if_exec:%[0-9]+]] = extractvalue { i1, i64 } [[if]], 1 +; OPT-LABEL: {{^}}define amdgpu_vs void @multi_else_break( +; OPT: main_body: +; OPT: LOOP.outer: +; OPT: LOOP: +; OPT: [[if:%[0-9]+]] = call { i1, i64 } @llvm.amdgcn.if( +; OPT: [[if_exec:%[0-9]+]] = extractvalue { i1, i64 } [[if]], 1 ; -; CHECK: Flow: +; OPT: Flow: ; ; Ensure two else.break calls, for both the inner and outer loops + +; OPT: call i64 @llvm.amdgcn.else.break(i64 [[if_exec]], +; OPT-NEXT: call i64 @llvm.amdgcn.else.break(i64 [[if_exec]], +; OPT-NEXT: call void @llvm.amdgcn.end.cf ; -; CHECK: call i64 @llvm.amdgcn.else.break(i64 [[if_exec]], -; CHECK-NEXT: call i64 @llvm.amdgcn.else.break(i64 [[if_exec]], -; CHECK-NEXT: call void @llvm.amdgcn.end.cf -; -; CHECK: Flow1: -define amdgpu_vs void @main(<4 x float> %vec, i32 %ub, i32 %cont) { +; OPT: Flow1: + +; GCN-LABEL: {{^}}multi_else_break: + +; GCN: [[OUTER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP.outer{{$}} + +; GCN: [[INNER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP{{$}} +; GCN: s_and_saveexec_b64 [[SAVE_BREAK:s\[[0-9]+:[0-9]+\]]], vcc + +; GCN: BB{{[0-9]+}}_{{[0-9]+}}: ; %Flow{{$}} +; GCN-NEXT: ; in Loop: Header=[[INNER_LOOP]] Depth=2 + +; Ensure extra or eliminated +; GCN-NEXT: s_or_b64 exec, exec, [[SAVE_BREAK]] +; GCN-NEXT: s_or_b64 [[OR_BREAK:s\[[0-9]+:[0-9]+\]]], [[SAVE_BREAK]], s{{\[[0-9]+:[0-9]+\]}} +; GCN-NEXT: s_andn2_b64 exec, exec, [[OR_BREAK]] +; GCN-NEXT: s_cbranch_execnz [[INNER_LOOP]] + +; GCN: ; BB#{{[0-9]+}}: ; %Flow1{{$}} +; GCN-NEXT: ; in Loop: Header=[[OUTER_LOOP]] Depth=1 + +; Ensure copy is eliminated +; GCN-NEXT: s_or_b64 exec, exec, [[OR_BREAK]] +; GCN-NEXT: s_or_b64 [[OUTER_OR_BREAK:s\[[0-9]+:[0-9]+\]]], [[SAVE_BREAK]], s{{\[[0-9]+:[0-9]+\]}} +; GCN-NEXT: s_andn2_b64 exec, exec, [[OUTER_OR_BREAK]] +; GCN-NEXT: s_cbranch_execnz [[OUTER_LOOP]] +define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) { main_body: br label %LOOP.outer @@ -38,4 +64,52 @@ ENDIF: ; preds = %LOOP br i1 %tmp51, label %LOOP, label %LOOP.outer } -attributes #0 = { nounwind readnone } +; OPT-LABEL: define void @multi_if_break_loop( +; OPT: llvm.amdgcn.break +; OPT: llvm.amdgcn.loop +; OPT: llvm.amdgcn.if.break +; OPT: llvm.amdgcn.if.break +; OPT: llvm.amdgcn.end.cf + +; GCN-LABEL: {{^}}multi_if_break_loop: +; GCN: s_mov_b64 [[BREAK_REG:s\[[0-9]+:[0-9]+\]]], 0{{$}} + +; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: ; %bb1{{$}} + +; Uses a copy intsead of an or +; GCN: s_mov_b64 [[COPY:s\[[0-9]+:[0-9]+\]]], [[BREAK_REG]] +; GCN: s_or_b64 [[BREAK_REG]], exec, [[COPY]] +define void @multi_if_break_loop(i32 %arg) #0 { +bb: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %tmp = sub i32 %id, %arg + br label %bb1 + +bb1: + %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ] + %lsr.iv.next = add i32 %lsr.iv, 1 + %cmp0 = icmp slt i32 %lsr.iv.next, 0 + %load0 = load volatile i32, i32 addrspace(1)* undef, align 4 + switch i32 %load0, label %bb9 [ + i32 0, label %case0 + i32 1, label %case1 + ] + +case0: + %load1 = load volatile i32, i32 addrspace(1)* undef, align 4 + %cmp1 = icmp slt i32 %tmp, %load1 + br i1 %cmp1, label %bb1, label %bb9 + +case1: + %load2 = load volatile i32, i32 addrspace(1)* undef, align 4 + %cmp2 = icmp slt i32 %tmp, %load2 + br i1 %cmp2, label %bb1, label %bb9 + +bb9: + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/operand-folding.ll b/test/CodeGen/AMDGPU/operand-folding.ll index d6fc65fa7e83..4e5ea4b86b77 100644 --- a/test/CodeGen/AMDGPU/operand-folding.ll +++ b/test/CodeGen/AMDGPU/operand-folding.ll @@ -109,6 +109,21 @@ entry: ret void } +; A subregister use operand should not be tied. +; CHECK-LABEL: {{^}}no_fold_tied_subregister: +; CHECK: buffer_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}} +; CHECK: v_mac_f32_e32 v[[LO]], 0x41200000, v[[HI]] +; CHECK: buffer_store_dword v[[LO]] +define void @no_fold_tied_subregister() { + %tmp1 = load volatile <2 x float>, <2 x float> addrspace(1)* undef + %tmp2 = extractelement <2 x float> %tmp1, i32 0 + %tmp3 = extractelement <2 x float> %tmp1, i32 1 + %tmp4 = fmul float %tmp3, 10.0 + %tmp5 = fadd float %tmp4, %tmp2 + store volatile float %tmp5, float addrspace(1)* undef + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #0 attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir b/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir new file mode 100644 index 000000000000..4584802ad5a7 --- /dev/null +++ b/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir @@ -0,0 +1,755 @@ +# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-optimize-exec-masking -o - %s | FileCheck %s + +--- | + target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64" + + define void @optimize_if_and_saveexec_xor(i32 %z, i32 %v) #0 { + main_body: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %cc = icmp eq i32 %id, 0 + %0 = call { i1, i64 } @llvm.amdgcn.if(i1 %cc) + %1 = extractvalue { i1, i64 } %0, 0 + %2 = extractvalue { i1, i64 } %0, 1 + br i1 %1, label %if, label %end + + if: ; preds = %main_body + %v.if = load volatile i32, i32 addrspace(1)* undef + br label %end + + end: ; preds = %if, %main_body + %r = phi i32 [ 4, %main_body ], [ %v.if, %if ] + call void @llvm.amdgcn.end.cf(i64 %2) + store i32 %r, i32 addrspace(1)* undef + ret void + } + + define void @optimize_if_and_saveexec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_or_saveexec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + + define void @optimize_if_and_saveexec_xor_valu_middle(i32 %z, i32 %v) #0 { + main_body: + %id = call i32 @llvm.amdgcn.workitem.id.x() + %cc = icmp eq i32 %id, 0 + %0 = call { i1, i64 } @llvm.amdgcn.if(i1 %cc) + %1 = extractvalue { i1, i64 } %0, 0 + %2 = extractvalue { i1, i64 } %0, 1 + store i32 %id, i32 addrspace(1)* undef + br i1 %1, label %if, label %end + + if: ; preds = %main_body + %v.if = load volatile i32, i32 addrspace(1)* undef + br label %end + + end: ; preds = %if, %main_body + %r = phi i32 [ 4, %main_body ], [ %v.if, %if ] + call void @llvm.amdgcn.end.cf(i64 %2) + store i32 %r, i32 addrspace(1)* undef + ret void + } + + define void @optimize_if_and_saveexec_xor_wrong_reg(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_and_saveexec_xor_modify_copy_to_exec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_and_saveexec_xor_live_out_setexec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_unknown_saveexec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_andn2_saveexec(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + define void @optimize_if_andn2_saveexec_no_commute(i32 %z, i32 %v) #0 { + main_body: + br i1 undef, label %if, label %end + + if: + br label %end + + end: + ret void + } + + ; Function Attrs: nounwind readnone + declare i32 @llvm.amdgcn.workitem.id.x() #1 + + declare { i1, i64 } @llvm.amdgcn.if(i1) + + declare void @llvm.amdgcn.end.cf(i64) + + + attributes #0 = { nounwind } + attributes #1 = { nounwind readnone } + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec_xor{{$}} +# CHECK: %sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 %vcc, implicit-def %exec, implicit-def %scc, implicit %exec +# CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: SI_MASK_BRANCH + +name: optimize_if_and_saveexec_xor +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec{{$}} +# CHECK: %sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 %vcc, implicit-def %exec, implicit-def %scc, implicit %exec +# CHECK-NEXT: SI_MASK_BRANCH + +name: optimize_if_and_saveexec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_or_saveexec{{$}} +# CHECK: %sgpr0_sgpr1 = S_OR_SAVEEXEC_B64 %vcc, implicit-def %exec, implicit-def %scc, implicit %exec +# CHECK-NEXT: SI_MASK_BRANCH + +name: optimize_if_or_saveexec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_OR_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec_xor_valu_middle +# CHECK: %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc +# CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET %vgpr0, undef %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) +# CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3 +# CHECK-NEXT: SI_MASK_BRANCH +name: optimize_if_and_saveexec_xor_valu_middle +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + BUFFER_STORE_DWORD_OFFSET %vgpr0, undef %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec_xor_wrong_reg{{$}} +# CHECK: %sgpr0_sgpr1 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc +# CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 undef %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: %exec = COPY %sgpr0_sgpr1 +# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec +name: optimize_if_and_saveexec_xor_wrong_reg +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr6 = S_MOV_B32 -1 + %sgpr7 = S_MOV_B32 61440 + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr0_sgpr1 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %sgpr0_sgpr1 = S_XOR_B64 undef %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term %sgpr0_sgpr1 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 , %sgpr4_sgpr5_sgpr6_sgpr7 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1, %sgpr4_sgpr5_sgpr6_sgpr7 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec_xor_modify_copy_to_exec{{$}} +# CHECK: %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc +# CHECK-NEXT: %sgpr2_sgpr3 = S_OR_B64 killed %sgpr2_sgpr3, 1, implicit-def %scc +# CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3 +# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec + +name: optimize_if_and_saveexec_xor_modify_copy_to_exec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %sgpr2_sgpr3 = S_OR_B64 killed %sgpr2_sgpr3, 1, implicit-def %scc + %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr0 = S_MOV_B32 0 + %sgpr1 = S_MOV_B32 1 + %sgpr2 = S_MOV_B32 -1 + %sgpr3 = S_MOV_B32 61440 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_and_saveexec_xor_live_out_setexec{{$}} +# CHECK: %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc +# CHECK-NEXT: %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: %exec = COPY %sgpr2_sgpr3 +# CHECK-NEXT: SI_MASK_BRANCH +name: optimize_if_and_saveexec_xor_live_out_setexec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %sgpr0_sgpr1 = S_XOR_B64 %sgpr2_sgpr3, killed %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1, %sgpr2_sgpr3 + S_SLEEP 0, implicit %sgpr2_sgpr3 + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... + +# CHECK-LABEL: name: optimize_if_unknown_saveexec{{$}} +# CHECK: %sgpr0_sgpr1 = COPY %exec +# CHECK: %sgpr2_sgpr3 = S_LSHR_B64 %sgpr0_sgpr1, killed %vcc_lo, implicit-def %scc +# CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3 +# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec + +name: optimize_if_unknown_saveexec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_LSHR_B64 %sgpr0_sgpr1, killed %vcc_lo, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_andn2_saveexec{{$}} +# CHECK: %sgpr0_sgpr1 = S_ANDN2_SAVEEXEC_B64 %vcc, implicit-def %exec, implicit-def %scc, implicit %exec +# CHECK-NEXT: SI_MASK_BRANCH + +name: optimize_if_andn2_saveexec +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_ANDN2_B64 %sgpr0_sgpr1, killed %vcc, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: optimize_if_andn2_saveexec_no_commute{{$}} +# CHECK: %sgpr2_sgpr3 = S_ANDN2_B64 killed %vcc, %sgpr0_sgpr1, implicit-def %scc +# CHECK-NEXT: %exec = COPY killed %sgpr2_sgpr3 +# CHECK-NEXT: SI_MASK_BRANCH %bb.2.end, implicit %exec +name: optimize_if_andn2_saveexec_no_commute +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%vgpr0' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.main_body: + successors: %bb.1.if, %bb.2.end + liveins: %vgpr0 + + %sgpr0_sgpr1 = COPY %exec + %vcc = V_CMP_EQ_I32_e64 0, killed %vgpr0, implicit %exec + %vgpr0 = V_MOV_B32_e32 4, implicit %exec + %sgpr2_sgpr3 = S_ANDN2_B64 killed %vcc, %sgpr0_sgpr1, implicit-def %scc + %exec = S_MOV_B64_term killed %sgpr2_sgpr3 + SI_MASK_BRANCH %bb.2.end, implicit %exec + S_BRANCH %bb.1.if + + bb.1.if: + successors: %bb.2.end + liveins: %sgpr0_sgpr1 + + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vgpr0 = BUFFER_LOAD_DWORD_OFFSET %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile load 4 from `i32 addrspace(1)* undef`) + + bb.2.end: + liveins: %vgpr0, %sgpr0_sgpr1 + + %exec = S_OR_B64 %exec, killed %sgpr0_sgpr1, implicit-def %scc + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into `i32 addrspace(1)* undef`) + S_ENDPGM + +... diff --git a/test/CodeGen/AMDGPU/or.ll b/test/CodeGen/AMDGPU/or.ll index 9b90ff798ca7..3e254850a934 100644 --- a/test/CodeGen/AMDGPU/or.ll +++ b/test/CodeGen/AMDGPU/or.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s @@ -62,6 +62,75 @@ define void @scalar_or_literal_i32(i32 addrspace(1)* %out, i32 %a) { ret void } +; FUNC-LABEL: {{^}}scalar_or_literal_i64: +; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-DAG: s_or_b32 s[[RES_HI:[0-9]+]], s[[HI]], 0xf237b +; SI-DAG: s_or_b32 s[[RES_LO:[0-9]+]], s[[LO]], 0x3039 +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_LO]] +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_HI]] +define void @scalar_or_literal_i64(i64 addrspace(1)* %out, i64 %a) { + %or = or i64 %a, 4261135838621753 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}scalar_or_literal_multi_use_i64: +; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0xf237b +; SI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x3039 +; SI: s_or_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} + +; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, s[[K_LO]] +; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, s[[K_HI]] +define void @scalar_or_literal_multi_use_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { + %or = or i64 %a, 4261135838621753 + store i64 %or, i64 addrspace(1)* %out + + %foo = add i64 %b, 4261135838621753 + store volatile i64 %foo, i64 addrspace(1)* undef + ret void +} + +; FUNC-LABEL: {{^}}scalar_or_inline_imm_i64: +; SI: s_load_dwordx2 s{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-NOT: or_b32 +; SI: s_or_b32 s[[VAL_LO]], s[[VAL_LO]], 63 +; SI-NOT: or_b32 +; SI: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[VAL_LO]] +; SI-NOT: or_b32 +; SI: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[VAL_HI]] +; SI-NOT: or_b32 +; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +define void @scalar_or_inline_imm_i64(i64 addrspace(1)* %out, i64 %a) { + %or = or i64 %a, 63 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}scalar_or_inline_imm_multi_use_i64: +; SI-NOT: or_b32 +; SI: s_or_b32 s{{[0-9]+}}, s{{[0-9]+}}, 63 +; SI-NOT: or_b32 +define void @scalar_or_inline_imm_multi_use_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { + %or = or i64 %a, 63 + store i64 %or, i64 addrspace(1)* %out + %foo = add i64 %b, 63 + store volatile i64 %foo, i64 addrspace(1)* undef + ret void +} + +; FUNC-LABEL: {{^}}scalar_or_neg_inline_imm_i64: +; SI-DAG: s_load_dword [[VAL:s[0-9]+]] +; SI-DAG: s_or_b32 [[VAL]], [[VAL]], -8 +; SI-DAG: v_mov_b32_e32 v[[V_HI:[0-9]+]], -1{{$}} +; SI-DAG: v_mov_b32_e32 v[[V_LO:[0-9]+]], [[VAL]] +; SI: buffer_store_dwordx2 v{{\[}}[[V_LO]]:[[V_HI]]{{\]}} +define void @scalar_or_neg_inline_imm_i64(i64 addrspace(1)* %out, i64 %a) { + %or = or i64 %a, -8 + store i64 %or, i64 addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}vector_or_literal_i32: ; SI: v_or_b32_e32 v{{[0-9]+}}, 0xffff, v{{[0-9]+}} define void @vector_or_literal_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %a, i32 addrspace(1)* %b) { @@ -113,11 +182,9 @@ define void @scalar_vector_or_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %a, } ; FUNC-LABEL: {{^}}vector_or_i64_loadimm: -; SI-DAG: s_mov_b32 [[LO_S_IMM:s[0-9]+]], 0xdf77987f -; SI-DAG: s_movk_i32 [[HI_S_IMM:s[0-9]+]], 0x146f ; SI-DAG: buffer_load_dwordx2 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, -; SI-DAG: v_or_b32_e32 {{v[0-9]+}}, [[LO_S_IMM]], v[[LO_VREG]] -; SI-DAG: v_or_b32_e32 {{v[0-9]+}}, [[HI_S_IMM]], v[[HI_VREG]] +; SI-DAG: v_or_b32_e32 {{v[0-9]+}}, 0xdf77987f, v[[LO_VREG]] +; SI-DAG: v_or_b32_e32 {{v[0-9]+}}, 0x146f, v[[HI_VREG]] ; SI: s_endpgm define void @vector_or_i64_loadimm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { %loada = load i64, i64 addrspace(1)* %a, align 8 @@ -129,8 +196,9 @@ define void @vector_or_i64_loadimm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, ; FIXME: The or 0 should really be removed. ; FUNC-LABEL: {{^}}vector_or_i64_imm: ; SI: buffer_load_dwordx2 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, -; SI: v_or_b32_e32 {{v[0-9]+}}, 8, v[[LO_VREG]] -; SI: v_or_b32_e32 {{v[0-9]+}}, 0, {{.*}} +; SI: v_or_b32_e32 v[[LO_RESULT:[0-9]+]], 8, v[[LO_VREG]] +; SI-NOT: v_or_b32_e32 {{v[0-9]+}}, 0 +; SI: buffer_store_dwordx2 v{{\[}}[[LO_RESULT]]:[[HI_VREG]]{{\]}} ; SI: s_endpgm define void @vector_or_i64_imm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { %loada = load i64, i64 addrspace(1)* %a, align 8 @@ -139,6 +207,32 @@ define void @vector_or_i64_imm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 ret void } +; FUNC-LABEL: {{^}}vector_or_i64_neg_inline_imm: +; SI-DAG: buffer_load_dword v[[LO_VREG:[0-9]+]] +; SI-DAG: v_or_b32_e32 v[[RES_LO:[0-9]+]], -8, v[[LO_VREG]] +; SI-DAG: v_mov_b32_e32 v[[RES_HI:[0-9]+]], -1{{$}} +; SI: buffer_store_dwordx2 v{{\[}}[[RES_LO]]:[[RES_HI]]{{\]}} +; SI: s_endpgm +define void @vector_or_i64_neg_inline_imm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { + %loada = load i64, i64 addrspace(1)* %a, align 8 + %or = or i64 %loada, -8 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}vector_or_i64_neg_literal: +; SI-DAG: buffer_load_dword v[[LO_VREG:[0-9]+]] +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, -1{{$}} +; SI-DAG: v_or_b32_e32 {{v[0-9]+}}, 0xffffff38, v[[LO_VREG]] +; SI: buffer_store_dwordx2 +; SI: s_endpgm +define void @vector_or_i64_neg_literal(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { + %loada = load i64, i64 addrspace(1)* %a, align 8 + %or = or i64 %loada, -200 + store i64 %or, i64 addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}trunc_i64_or_to_i32: ; SI: s_load_dword s[[SREG0:[0-9]+]] ; SI: s_load_dword s[[SREG1:[0-9]+]] diff --git a/test/CodeGen/AMDGPU/private-access-no-objects.ll b/test/CodeGen/AMDGPU/private-access-no-objects.ll new file mode 100644 index 000000000000..2894730eccb1 --- /dev/null +++ b/test/CodeGen/AMDGPU/private-access-no-objects.ll @@ -0,0 +1,56 @@ +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=OPT %s +; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=iceland -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s +; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=OPTNONE %s + +; There are no stack objects, but still a private memory access. The +; private access regiters need to be correctly initialized anyway, and +; shifted down to the end of the used registers. + +; GCN-LABEL: {{^}}store_to_undef: +; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1] +; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3] +; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}} +; OPT: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}} + +; -O0 should assume spilling, so the input scratch resource descriptor +; -should be used directly without any copies. + +; OPTNONE-NOT: s_mov_b32 +; OPTNONE: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[0:3], s7 offen{{$}} +define void @store_to_undef() #0 { + store volatile i32 0, i32* undef + ret void +} + +; GCN-LABEL: {{^}}store_to_inttoptr: +; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1] +; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3] +; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}} +; OPT: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}} +define void @store_to_inttoptr() #0 { + store volatile i32 0, i32* inttoptr (i32 123 to i32*) + ret void +} + +; GCN-LABEL: {{^}}load_from_undef: +; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1] +; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3] +; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}} +; OPT: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}} +define void @load_from_undef() #0 { + %ld = load volatile i32, i32* undef + ret void +} + +; GCN-LABEL: {{^}}load_from_inttoptr: +; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1] +; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3] +; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}} +; OPT: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}} +define void @load_from_inttoptr() #0 { + %ld = load volatile i32, i32* inttoptr (i32 123 to i32*) + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/private-element-size.ll b/test/CodeGen/AMDGPU/private-element-size.ll index cd8fb22e620a..de9a8f755122 100644 --- a/test/CodeGen/AMDGPU/private-element-size.ll +++ b/test/CodeGen/AMDGPU/private-element-size.ll @@ -10,27 +10,27 @@ ; HSA-ELT4: private_element_size = 1 -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 ; HSA-ELT16-DAG: buffer_load_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:8 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:24 ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:12{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:20{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:28{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:4{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:8{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:12{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:16{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:20{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:24{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:28{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} @@ -59,44 +59,44 @@ entry: ; HSA-ELT8: private_element_size = 2 ; HSA-ELT4: private_element_size = 1 -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:32 -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:48 +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:32 +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:48 ; HSA-ELT16-DAG: buffer_load_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT16-DAG: buffer_load_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:32 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:40 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:48 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:56 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:8 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:24 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:32 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:40 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:48 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:56 ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:12{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:20{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:28{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:32{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:36{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:40{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:44{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:48{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:52{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:56{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:60{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:4{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:8{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:12{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:16{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:20{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:24{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:28{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:32{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:36{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:40{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:44{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:48{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:52{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:56{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:60{{$}} ; HSA-ELT4-DAG: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT4-DAG: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} @@ -130,16 +130,16 @@ entry: ; HSA-ELT8: private_element_size = 2 ; HSA-ELT4: private_element_size = 1 -; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8 +; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:8 ; HSA-ELTGE8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:12{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:4{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:8{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:12{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} @@ -166,16 +166,16 @@ entry: ; HSA-ELT8: private_element_size = 2 ; HSA-ELT4: private_element_size = 1 -; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8 +; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELTGE8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:8 ; HSA-ELTGE8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:12{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:4{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:8{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:12{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} @@ -202,27 +202,27 @@ entry: ; HSA-ELT8: private_element_size = 2 ; HSA-ELT4: private_element_size = 1 -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT16-DAG: buffer_store_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 ; HSA-ELT16-DAG: buffer_load_dwordx4 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16 -; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9{{$}} +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:8 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:16 +; HSA-ELT8-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, off, s[0:3], s9 offset:24 ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen ; HSA-ELT8: buffer_load_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}, s[0:3], s9 offen -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:8{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:12{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:16{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:20{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:24{{$}} -; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:28{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:4{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:8{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:12{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:16{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:20{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:24{{$}} +; HSA-ELT4-DAG: buffer_store_dword {{v[0-9]+}}, off, s[0:3], s9 offset:28{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen{{$}} ; HSA-ELT4: buffer_load_dword {{v[0-9]+}}, v{{[0-9]+}}, s[0:3], s9 offen offset:4{{$}} diff --git a/test/CodeGen/AMDGPU/private-memory-r600.ll b/test/CodeGen/AMDGPU/private-memory-r600.ll index 883bdc1ce265..3e1796959aa6 100644 --- a/test/CodeGen/AMDGPU/private-memory-r600.ll +++ b/test/CodeGen/AMDGPU/private-memory-r600.ll @@ -297,4 +297,4 @@ define void @ptrtoint(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { ; OPT: !0 = !{i32 0, i32 2048} -attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="2" } +attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,2" } diff --git a/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll b/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll new file mode 100644 index 000000000000..f00b3de857f3 --- /dev/null +++ b/test/CodeGen/AMDGPU/promote-alloca-addrspacecast.ll @@ -0,0 +1,21 @@ +; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -mcpu=kaveri -amdgpu-promote-alloca < %s | FileCheck %s + +; The types of the users of the addrspacecast should not be changed. + +; CHECK-LABEL: @invalid_bitcast_addrspace( +; CHECK: getelementptr inbounds [256 x [1 x i32]], [256 x [1 x i32]] addrspace(3)* @invalid_bitcast_addrspace.data, i32 0, i32 %14 +; CHECK: bitcast [1 x i32] addrspace(3)* %{{[0-9]+}} to half addrspace(3)* +; CHECK: addrspacecast half addrspace(3)* %tmp to half addrspace(4)* +; CHECK: bitcast half addrspace(4)* %tmp1 to <2 x i16> addrspace(4)* +define amdgpu_kernel void @invalid_bitcast_addrspace() #0 { +entry: + %data = alloca [1 x i32], align 4 + %tmp = bitcast [1 x i32]* %data to half* + %tmp1 = addrspacecast half* %tmp to half addrspace(4)* + %tmp2 = bitcast half addrspace(4)* %tmp1 to <2 x i16> addrspace(4)* + %tmp3 = load <2 x i16>, <2 x i16> addrspace(4)* %tmp2, align 2 + %tmp4 = bitcast <2 x i16> %tmp3 to <2 x half> + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll b/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll index 6a9ec31696d2..4c3c15dac0d1 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-invariant-markers.ll @@ -1,7 +1,7 @@ ; RUN: llc -march=amdgcn -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s -declare {}* @llvm.invariant.start(i64, i8* nocapture) #0 -declare void @llvm.invariant.end({}*, i64, i8* nocapture) #0 +declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) #0 +declare void @llvm.invariant.end.p0i8({}*, i64, i8* nocapture) #0 declare i8* @llvm.invariant.group.barrier(i8*) #1 ; GCN-LABEL: {{^}}use_invariant_promotable_lds: @@ -14,8 +14,8 @@ bb: %tmp2 = getelementptr inbounds i32, i32 addrspace(1)* %arg, i64 1 %tmp3 = load i32, i32 addrspace(1)* %tmp2 store i32 %tmp3, i32* %tmp - %tmp4 = call {}* @llvm.invariant.start(i64 4, i8* %tmp1) #0 - call void @llvm.invariant.end({}* %tmp4, i64 4, i8* %tmp1) #0 + %tmp4 = call {}* @llvm.invariant.start.p0i8(i64 4, i8* %tmp1) #0 + call void @llvm.invariant.end.p0i8({}* %tmp4, i64 4, i8* %tmp1) #0 %tmp5 = call i8* @llvm.invariant.group.barrier(i8* %tmp1) #1 ret void } diff --git a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll index 01ecb638b033..9cea1a23ea98 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll @@ -61,5 +61,5 @@ define void @promote_with_objectsize(i32 addrspace(1)* %out) #0 { ret void } -attributes #0 = { nounwind "amdgpu-max-work-group-size"="64" "amdgpu-max-waves-per-eu"="3" } +attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,3" } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll b/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll index 7c5a5182bc8e..8ba849e5f884 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-no-opts.ll @@ -34,5 +34,5 @@ entry: ret void } -attributes #0 = { nounwind "amdgpu-max-work-group-size"="64" } -attributes #1 = { nounwind optnone noinline "amdgpu-max-work-group-size"="64" } +attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" } +attributes #1 = { nounwind optnone noinline "amdgpu-flat-work-group-size"="64,64" } diff --git a/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll b/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll index 46fe307a17fe..468a789e4a67 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-padding-size-estimate.ll @@ -127,4 +127,4 @@ entry: ret void } -attributes #0 = { nounwind "amdgpu-max-work-group-size"="64" } +attributes #0 = { nounwind "amdgpu-flat-work-group-size"="64,64" "amdgpu-waves-per-eu"="1,7" } diff --git a/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll b/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll index 307eca712cc8..3bcbb4f986b7 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-stored-pointer-value.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -mattr=+promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s -; RUN: llc -march=amdgcn -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mattr=+promote-alloca,+max-private-element-size-4 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mattr=-promote-alloca,+max-private-element-size-4 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; Pointer value is stored in a candidate for LDS usage. diff --git a/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll b/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll index 857e547aa03b..2e7527dbdbc4 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-to-lds-icmp.ll @@ -61,4 +61,4 @@ define void @lds_promoted_alloca_icmp_unknown_ptr(i32 addrspace(1)* %out, i32 %a declare i32* @get_unknown_pointer() #0 -attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="1" } +attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" } diff --git a/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll b/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll index a0ad564a6c8f..0462a351c39b 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-to-lds-phi.ll @@ -201,4 +201,4 @@ for.body: ; preds = %for.body, %for.body declare i32* @get_unknown_pointer() #0 -attributes #0 = { nounwind "amdgpu-max-waves-per-eu"="1" } +attributes #0 = { nounwind "amdgpu-waves-per-eu"="1,1" } diff --git a/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll b/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll index bb13adb19852..34d274df7387 100644 --- a/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll +++ b/test/CodeGen/AMDGPU/promote-alloca-to-lds-select.ll @@ -129,5 +129,5 @@ bb: ret void } -attributes #0 = { norecurse nounwind "amdgpu-max-waves-per-eu"="1" } +attributes #0 = { norecurse nounwind "amdgpu-waves-per-eu"="1,1" } attributes #1 = { norecurse nounwind }
\ No newline at end of file diff --git a/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll b/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll new file mode 100644 index 000000000000..d5bf9c88b780 --- /dev/null +++ b/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll @@ -0,0 +1,28 @@ +; RUN: llc -filetype=obj -march=r600 -mcpu=cypress -verify-machineinstrs < %s | llvm-readobj -relocations -symbols | FileCheck %s + +@arr = internal unnamed_addr addrspace(2) constant [4 x i32] [i32 4, i32 5, i32 6, i32 7], align 4 + +; CHECK: Relocations [ +; CHECK: Section (3) .rel.text { +; CHECK: 0x58 R_AMDGPU_ABS32 .text 0x0 +; CHECK: } +; CHECK: ] + +; CHECK: Symbol { +; CHECK: Name: arr (11) +; CHECK: Value: 0x70 +; CHECK: Size: 16 +; CHECK: Binding: Local (0x0) +; CHECK: Type: Object (0x1) +; CHECK: Other: 0 +; CHECK: Section: .text (0x2) +; CHECK: } +define amdgpu_kernel void @test_constant_array_fixup(i32 addrspace(1)* nocapture %out, i32 %idx) #0 { +entry: + %arrayidx = getelementptr inbounds [4 x i32], [4 x i32] addrspace(2)* @arr, i32 0, i32 %idx + %val = load i32, i32 addrspace(2)* %arrayidx + store i32 %val, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/r600-export-fix.ll b/test/CodeGen/AMDGPU/r600-export-fix.ll index 7d86f9e3b3f1..665feeb678b7 100644 --- a/test/CodeGen/AMDGPU/r600-export-fix.ll +++ b/test/CodeGen/AMDGPU/r600-export-fix.ll @@ -3,9 +3,9 @@ ;CHECK: EXPORT T{{[0-9]}}.XYZW ;CHECK: EXPORT T{{[0-9]}}.0000 ;CHECK: EXPORT T{{[0-9]}}.0000 -;CHECK: EXPORT T{{[0-9]}}.0XYZ +;CHECK: EXPORT T{{[0-9]}}.0YZW ;CHECK: EXPORT T{{[0-9]}}.XYZW -;CHECK: EXPORT T{{[0-9]}}.YZ00 +;CHECK: EXPORT T{{[0-9]}}.XY00 ;CHECK: EXPORT T{{[0-9]}}.0000 ;CHECK: EXPORT T{{[0-9]}}.0000 diff --git a/test/CodeGen/AMDGPU/r600.bitcast.ll b/test/CodeGen/AMDGPU/r600.bitcast.ll new file mode 100644 index 000000000000..49441ee8d186 --- /dev/null +++ b/test/CodeGen/AMDGPU/r600.bitcast.ll @@ -0,0 +1,107 @@ +; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s + +; This test just checks that the compiler doesn't crash. + + +; FUNC-LABEL: {{^}}i8ptr_v16i8ptr: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.XYZW]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_128 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @i8ptr_v16i8ptr(<16 x i8> addrspace(1)* %out, i8 addrspace(1)* %in) { +entry: + %0 = bitcast i8 addrspace(1)* %in to <16 x i8> addrspace(1)* + %1 = load <16 x i8>, <16 x i8> addrspace(1)* %0 + store <16 x i8> %1, <16 x i8> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}f32_to_v2i16: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.[XYZW]]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_32 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @f32_to_v2i16(<2 x i16> addrspace(1)* %out, float addrspace(1)* %in) nounwind { + %load = load float, float addrspace(1)* %in, align 4 + %bc = bitcast float %load to <2 x i16> + store <2 x i16> %bc, <2 x i16> addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}v2i16_to_f32: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.[XYZW]]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_32 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @v2i16_to_f32(float addrspace(1)* %out, <2 x i16> addrspace(1)* %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4 + %bc = bitcast <2 x i16> %load to float + store float %bc, float addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}v4i8_to_i32: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.[XYZW]]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_32 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @v4i8_to_i32(i32 addrspace(1)* %out, <4 x i8> addrspace(1)* %in) nounwind { + %load = load <4 x i8>, <4 x i8> addrspace(1)* %in, align 4 + %bc = bitcast <4 x i8> %load to i32 + store i32 %bc, i32 addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}i32_to_v4i8: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.[XYZW]]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_32 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @i32_to_v4i8(<4 x i8> addrspace(1)* %out, i32 addrspace(1)* %in) nounwind { + %load = load i32, i32 addrspace(1)* %in, align 4 + %bc = bitcast i32 %load to <4 x i8> + store <4 x i8> %bc, <4 x i8> addrspace(1)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}v2i16_to_v4i8: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.[XYZW]]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_32 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @v2i16_to_v4i8(<4 x i8> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) nounwind { + %load = load <2 x i16>, <2 x i16> addrspace(1)* %in, align 4 + %bc = bitcast <2 x i16> %load to <4 x i8> + store <4 x i8> %bc, <4 x i8> addrspace(1)* %out, align 4 + ret void +} + +; This just checks for crash in BUILD_VECTOR/EXTRACT_ELEMENT combine +; the stack manipulation is tricky to follow +; TODO: This should only use one load +; FUNC-LABEL: {{^}}v4i16_extract_i8: +; EG: MEM_RAT MSKOR {{T[0-9]+\.XW}}, [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_16 +; EG: VTX_READ_16 +; EG-DAG: BFE_UINT +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @v4i16_extract_i8(i8 addrspace(1)* %out, <4 x i16> addrspace(1)* %in) nounwind { + %load = load <4 x i16>, <4 x i16> addrspace(1)* %in, align 2 + %bc = bitcast <4 x i16> %load to <8 x i8> + %element = extractelement <8 x i8> %bc, i32 5 + store i8 %element, i8 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}bitcast_v2i32_to_f64: +; EG: MEM_RAT_CACHELESS STORE_RAW [[DATA:T[0-9]+\.XY]], [[ST_PTR:T[0-9]+\.[XYZW]]] +; EG: VTX_READ_64 [[DATA]], [[LD_PTR:T[0-9]+\.[XYZW]]] +; EG-DAG: MOV {{[\* ]*}}[[LD_PTR]], KC0[2].Z +; EG-DAG: LSHR {{[\* ]*}}[[ST_PTR]], KC0[2].Y, literal +define void @bitcast_v2i32_to_f64(double addrspace(1)* %out, <2 x i32> addrspace(1)* %in) { + %val = load <2 x i32>, <2 x i32> addrspace(1)* %in, align 8 + %bc = bitcast <2 x i32> %val to double + store double %bc, double addrspace(1)* %out, align 8 + ret void +} + diff --git a/test/CodeGen/AMDGPU/r600.work-item-intrinsics.ll b/test/CodeGen/AMDGPU/r600.work-item-intrinsics.ll index ff248a89cedc..a34a48e3b7ba 100644 --- a/test/CodeGen/AMDGPU/r600.work-item-intrinsics.ll +++ b/test/CodeGen/AMDGPU/r600.work-item-intrinsics.ll @@ -78,22 +78,6 @@ define void @test_implicit_dyn(i32 addrspace(1)* %out, i32 %in) #1 { ret void } - - -; DEPRECATED but R600 only - -; FUNC-LABEL: {{^}}workdim: -; EG: MEM_RAT_CACHELESS STORE_RAW [[VAL:T[0-9]+\.X]] -; EG: MOV {{\*? *}}[[VAL]], KC0[2].Z -define void @workdim (i32 addrspace(1)* %out) { -entry: - %0 = call i32 @llvm.r600.read.workdim() #0 - store i32 %0, i32 addrspace(1)* %out - ret void -} - -declare i32 @llvm.r600.read.workdim() #0 - declare i8 addrspace(7)* @llvm.r600.implicitarg.ptr() #0 declare i32 @llvm.r600.read.tgid.x() #0 diff --git a/test/CodeGen/AMDGPU/rcp-pattern.ll b/test/CodeGen/AMDGPU/rcp-pattern.ll index 27a88f7b59e7..58d411772f14 100644 --- a/test/CodeGen/AMDGPU/rcp-pattern.ll +++ b/test/CodeGen/AMDGPU/rcp-pattern.ll @@ -76,8 +76,22 @@ define void @rcp_fabs_pat_f32(float addrspace(1)* %out, float %src) #0 { ret void } -; FIXME: fneg folded into constant 1 +; FUNC-LABEL: {{^}}neg_rcp_pat_f32: +; GCN: s_load_dword [[SRC:s[0-9]+]] +; GCN: v_rcp_f32_e64 [[RCP:v[0-9]+]], -[[SRC]] +; GCN: buffer_store_dword [[RCP]] + +; EG: RECIP_IEEE +define void @neg_rcp_pat_f32(float addrspace(1)* %out, float %src) #0 { + %rcp = fdiv float -1.0, %src + store float %rcp, float addrspace(1)* %out, align 4 + ret void +} + ; FUNC-LABEL: {{^}}rcp_fabs_fneg_pat_f32: +; GCN: s_load_dword [[SRC:s[0-9]+]] +; GCN: v_rcp_f32_e64 [[RCP:v[0-9]+]], -|[[SRC]]| +; GCN: buffer_store_dword [[RCP]] define void @rcp_fabs_fneg_pat_f32(float addrspace(1)* %out, float %src) #0 { %src.fabs = call float @llvm.fabs.f32(float %src) %src.fabs.fneg = fsub float -0.0, %src.fabs @@ -86,8 +100,26 @@ define void @rcp_fabs_fneg_pat_f32(float addrspace(1)* %out, float %src) #0 { ret void } +; FUNC-LABEL: {{^}}rcp_fabs_fneg_pat_multi_use_f32: +; GCN: s_load_dword [[SRC:s[0-9]+]] +; GCN: v_rcp_f32_e64 [[RCP:v[0-9]+]], -|[[SRC]]| +; GCN: v_mul_f32_e64 [[MUL:v[0-9]+]], [[SRC]], -|[[SRC]]| +; GCN: buffer_store_dword [[RCP]] +; GCN: buffer_store_dword [[MUL]] +define void @rcp_fabs_fneg_pat_multi_use_f32(float addrspace(1)* %out, float %src) #0 { + %src.fabs = call float @llvm.fabs.f32(float %src) + %src.fabs.fneg = fsub float -0.0, %src.fabs + %rcp = fdiv float 1.0, %src.fabs.fneg + store volatile float %rcp, float addrspace(1)* %out, align 4 + + %other = fmul float %src, %src.fabs.fneg + store volatile float %other, float addrspace(1)* %out, align 4 + ret void +} + declare float @llvm.fabs.f32(float) #1 +declare float @llvm.sqrt.f32(float) #1 attributes #0 = { nounwind "unsafe-fp-math"="false" } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/read_register.ll b/test/CodeGen/AMDGPU/read_register.ll index 58a9e34b77f2..601a0adb8122 100644 --- a/test/CodeGen/AMDGPU/read_register.ll +++ b/test/CodeGen/AMDGPU/read_register.ll @@ -3,9 +3,11 @@ declare i32 @llvm.read_register.i32(metadata) #0 declare i64 @llvm.read_register.i64(metadata) #0 +; FIXME: Should be able to eliminate copy ; CHECK-LABEL: {{^}}test_read_m0: ; CHECK: s_mov_b32 m0, -1 -; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], m0 +; CHECK: s_mov_b32 [[COPY_M0:s[0-9]+]], m0 +; CHECK: v_mov_b32_e32 [[COPY:v[0-9]+]], [[COPY_M0]] ; CHECK: buffer_store_dword [[COPY]] define void @test_read_m0(i32 addrspace(1)* %out) #0 { store volatile i32 0, i32 addrspace(3)* undef diff --git a/test/CodeGen/AMDGPU/rename-independent-subregs.mir b/test/CodeGen/AMDGPU/rename-independent-subregs.mir index 2dd21ca51e46..b928bc7086bb 100644 --- a/test/CodeGen/AMDGPU/rename-independent-subregs.mir +++ b/test/CodeGen/AMDGPU/rename-independent-subregs.mir @@ -1,30 +1,70 @@ -# RUN: llc -march=amdgcn -run-pass rename-independent-subregs -o - %s | FileCheck %s +# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass simple-register-coalescing,rename-independent-subregs -o - %s | FileCheck %s --- | define void @test0() { ret void } + define void @test1() { ret void } ... --- # In the test below we have two independent def+use pairs of subregister1 which # can be moved to a new virtual register. The third def of sub1 however is used # in combination with sub0 and needs to stay with the original vreg. # CHECK-LABEL: name: test0 -# CHECK: S_NOP 0, implicit-def undef %0:sub0 -# CHECK: S_NOP 0, implicit-def undef %2:sub1 -# CHECK: S_NOP 0, implicit %2:sub1 -# CHECK: S_NOP 0, implicit-def undef %1:sub1 -# CHECK: S_NOP 0, implicit %1:sub1 -# CHECK: S_NOP 0, implicit-def %0:sub1 +# CHECK: S_NOP 0, implicit-def undef %0.sub0 +# CHECK: S_NOP 0, implicit-def undef %2.sub1 +# CHECK: S_NOP 0, implicit %2.sub1 +# CHECK: S_NOP 0, implicit-def undef %1.sub1 +# CHECK: S_NOP 0, implicit %1.sub1 +# CHECK: S_NOP 0, implicit-def %0.sub1 # CHECK: S_NOP 0, implicit %0 name: test0 -isSSA: true registers: - { id: 0, class: sreg_128 } body: | bb.0: - S_NOP 0, implicit-def undef %0:sub0 - S_NOP 0, implicit-def %0:sub1 - S_NOP 0, implicit %0:sub1 - S_NOP 0, implicit-def %0:sub1 - S_NOP 0, implicit %0:sub1 - S_NOP 0, implicit-def %0:sub1 + S_NOP 0, implicit-def undef %0.sub0 + S_NOP 0, implicit-def %0.sub1 + S_NOP 0, implicit %0.sub1 + S_NOP 0, implicit-def %0.sub1 + S_NOP 0, implicit %0.sub1 + S_NOP 0, implicit-def %0.sub1 S_NOP 0, implicit %0 ... +--- +# Test for a bug where we would incorrectly query liveness at the instruction +# index in rewriteOperands(). This should pass the verifier afterwards. +# CHECK-LABEL: test1 +# CHECK: bb.0 +# CHECK: S_NOP 0, implicit-def undef %2.sub2 +# CHECK: bb.1 +# CHECK: S_NOP 0, implicit-def %2.sub1 +# CHECK-NEXT: S_NOP 0, implicit-def %2.sub3 +# CHECK-NEXT: S_NOP 0, implicit %2 +# CHECK-NEXT: S_NOP 0, implicit-def undef %0.sub0 +# CHECK-NEXT: S_NOP 0, implicit %2.sub1 +# CHECK-NEXT: S_NOP 0, implicit %0.sub0 +# CHECK: bb.2 +# CHECK: S_NOP 0, implicit %2.sub +name: test1 +registers: + - { id: 0, class: sreg_128 } + - { id: 1, class: sreg_128 } +body: | + bb.0: + successors: %bb.1, %bb.2 + S_NOP 0, implicit-def undef %0.sub2 + S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc + S_BRANCH %bb.2 + + bb.1: + S_NOP 0, implicit-def %0.sub1 + S_NOP 0, implicit-def %0.sub3 + %1 = COPY %0 + S_NOP 0, implicit %1 + + S_NOP 0, implicit-def %1.sub0 + S_NOP 0, implicit %1.sub1 + S_NOP 0, implicit %1.sub0 + + bb.2: + S_NOP 0, implicit %0.sub2 + +... diff --git a/test/CodeGen/AMDGPU/ret.ll b/test/CodeGen/AMDGPU/ret.ll index 915c4383ff49..515203fad4cb 100644 --- a/test/CodeGen/AMDGPU/ret.ll +++ b/test/CodeGen/AMDGPU/ret.ll @@ -6,7 +6,7 @@ declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float ; GCN-LABEL: {{^}}vgpr: ; GCN: v_mov_b32_e32 v1, v0 ; GCN-DAG: v_add_f32_e32 v0, 1.0, v1 -; GCN-DAG: exp 15, 0, 1, 1, 1, v1, v1, v1, v1 +; GCN-DAG: exp mrt0 v1, v1, v1, v1 done compr vm ; GCN: s_waitcnt expcnt(0) ; GCN-NOT: s_endpgm define amdgpu_vs {float, float} @vgpr([9 x <16 x i8>] addrspace(2)* byval, i32 inreg, i32 inreg, float) { @@ -18,12 +18,14 @@ define amdgpu_vs {float, float} @vgpr([9 x <16 x i8>] addrspace(2)* byval, i32 i } ; GCN-LABEL: {{^}}vgpr_literal: -; GCN: exp 15, 0, 1, 1, 1, v0, v0, v0, v0 -; GCN: s_waitcnt expcnt(0) +; GCN: v_mov_b32_e32 v4, v0 +; GCN: exp mrt0 v4, v4, v4, v4 done compr vm + ; GCN-DAG: v_mov_b32_e32 v0, 1.0 ; GCN-DAG: v_mov_b32_e32 v1, 2.0 ; GCN-DAG: v_mov_b32_e32 v2, 4.0 ; GCN-DAG: v_mov_b32_e32 v3, -1.0 +; GCN: s_waitcnt expcnt(0) ; GCN-NOT: s_endpgm define amdgpu_vs {float, float, float, float} @vgpr_literal([9 x <16 x i8>] addrspace(2)* byval, i32 inreg, i32 inreg, float) { call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %3, float %3, float %3, float %3) @@ -42,7 +44,6 @@ define amdgpu_vs {float, float, float, float} @vgpr_literal([9 x <16 x i8>] addr ; GCN: v_mov_b32_e32 v3, v4 ; GCN: v_mov_b32_e32 v4, v6 ; GCN-NOT: s_endpgm -attributes #0 = { "InitialPSInputAddr"="0" } define amdgpu_ps {float, float, float, float, float} @vgpr_ps_addr0([9 x <16 x i8>] addrspace(2)* byval, i32 inreg, i32 inreg, <2 x i32>, <2 x i32>, <2 x i32>, <3 x i32>, <2 x i32>, <2 x i32>, <2 x i32>, float, float, float, float, float, float, float, float, float) #0 { %i0 = extractelement <2 x i32> %4, i32 0 %i1 = extractelement <2 x i32> %4, i32 1 @@ -208,7 +209,7 @@ define amdgpu_vs {i32, i32, i32, i32} @sgpr_literal([9 x <16 x i8>] addrspace(2) ; GCN-LABEL: {{^}}both: ; GCN: v_mov_b32_e32 v1, v0 -; GCN-DAG: exp 15, 0, 1, 1, 1, v1, v1, v1, v1 +; GCN-DAG: exp mrt0 v1, v1, v1, v1 done compr vm ; GCN-DAG: v_add_f32_e32 v0, 1.0, v1 ; GCN-DAG: s_add_i32 s0, s3, 2 ; GCN-DAG: s_mov_b32 s1, s2 @@ -229,14 +230,18 @@ define amdgpu_vs {float, i32, float, i32, i32} @both([9 x <16 x i8>] addrspace(2 ; GCN-LABEL: {{^}}structure_literal: -; GCN: exp 15, 0, 1, 1, 1, v0, v0, v0, v0 -; GCN: s_waitcnt expcnt(0) +; GCN: v_mov_b32_e32 v3, v0 +; GCN: exp mrt0 v3, v3, v3, v3 done compr vm + ; GCN-DAG: v_mov_b32_e32 v0, 1.0 ; GCN-DAG: s_mov_b32 s0, 2 ; GCN-DAG: s_mov_b32 s1, 3 ; GCN-DAG: v_mov_b32_e32 v1, 2.0 ; GCN-DAG: v_mov_b32_e32 v2, 4.0 +; GCN: s_waitcnt expcnt(0) define amdgpu_vs {{float, i32}, {i32, <2 x float>}} @structure_literal([9 x <16 x i8>] addrspace(2)* byval, i32 inreg, i32 inreg, float) { call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %3, float %3, float %3, float %3) ret {{float, i32}, {i32, <2 x float>}} {{float, i32} {float 1.0, i32 2}, {i32, <2 x float>} {i32 3, <2 x float> <float 2.0, float 4.0>}} } + +attributes #0 = { nounwind "InitialPSInputAddr"="0" } diff --git a/test/CodeGen/AMDGPU/ret_jump.ll b/test/CodeGen/AMDGPU/ret_jump.ll index f7380cd96921..51ca60492414 100644 --- a/test/CodeGen/AMDGPU/ret_jump.ll +++ b/test/CodeGen/AMDGPU/ret_jump.ll @@ -5,7 +5,7 @@ ; Mask should be in original state after executed unreachable block ; GCN-LABEL: {{^}}main: -; GCN: s_cbranch_vccnz [[RET_BB:BB[0-9]+_[0-9]+]] +; GCN: s_cbranch_scc1 [[RET_BB:BB[0-9]+_[0-9]+]] ; GCN: s_and_saveexec_b64 [[SAVE_EXEC:s\[[0-9]+:[0-9]+\]]], vcc ; GCN-NEXT: s_xor_b64 [[XOR_EXEC:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE_EXEC]] diff --git a/test/CodeGen/AMDGPU/rsq.ll b/test/CodeGen/AMDGPU/rsq.ll index 8192b861b602..699440c3efbf 100644 --- a/test/CodeGen/AMDGPU/rsq.ll +++ b/test/CodeGen/AMDGPU/rsq.ll @@ -72,3 +72,67 @@ define void @rsqrt_fmul(float addrspace(1)* %out, float addrspace(1)* %in) { store float %z, float addrspace(1)* %out.gep ret void } + +; SI-LABEL: {{^}}neg_rsq_f32: +; SI-SAFE: v_sqrt_f32_e32 [[SQRT:v[0-9]+]], v{{[0-9]+}} +; SI-SAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] +; SI-SAFE: buffer_store_dword [[RSQ]] + +; SI-UNSAFE: v_rsq_f32_e32 [[RSQ:v[0-9]+]], v{{[0-9]+}} +; SI-UNSAFE: v_xor_b32_e32 [[NEG_RSQ:v[0-9]+]], 0x80000000, [[RSQ]] +; SI-UNSAFE: buffer_store_dword [[NEG_RSQ]] +define void @neg_rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind { + %val = load float, float addrspace(1)* %in, align 4 + %sqrt = call float @llvm.sqrt.f32(float %val) + %div = fdiv float -1.0, %sqrt + store float %div, float addrspace(1)* %out, align 4 + ret void +} + +; SI-LABEL: {{^}}neg_rsq_f64: +; SI-SAFE: v_sqrt_f64_e32 +; SI-SAFE: v_div_scale_f64 + +; SI-UNSAFE: v_sqrt_f64_e32 [[SQRT:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}} +; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]] +; SI-UNSAFE: buffer_store_dwordx2 [[RCP]] +define void @neg_rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind { + %val = load double, double addrspace(1)* %in, align 4 + %sqrt = call double @llvm.sqrt.f64(double %val) + %div = fdiv double -1.0, %sqrt + store double %div, double addrspace(1)* %out, align 4 + ret void +} + +; SI-LABEL: {{^}}neg_rsq_neg_f32: +; SI-SAFE: v_sqrt_f32_e64 [[SQRT:v[0-9]+]], -v{{[0-9]+}} +; SI-SAFE: v_rcp_f32_e64 [[RSQ:v[0-9]+]], -[[SQRT]] +; SI-SAFE: buffer_store_dword [[RSQ]] + +; SI-UNSAFE: v_rsq_f32_e64 [[RSQ:v[0-9]+]], -v{{[0-9]+}} +; SI-UNSAFE: v_xor_b32_e32 [[NEG_RSQ:v[0-9]+]], 0x80000000, [[RSQ]] +; SI-UNSAFE: buffer_store_dword [[NEG_RSQ]] +define void @neg_rsq_neg_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind { + %val = load float, float addrspace(1)* %in, align 4 + %val.fneg = fsub float -0.0, %val + %sqrt = call float @llvm.sqrt.f32(float %val.fneg) + %div = fdiv float -1.0, %sqrt + store float %div, float addrspace(1)* %out, align 4 + ret void +} + +; SI-LABEL: {{^}}neg_rsq_neg_f64: +; SI-SAFE: v_sqrt_f64_e64 v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}} +; SI-SAFE: v_div_scale_f64 + +; SI-UNSAFE: v_sqrt_f64_e64 [[SQRT:v\[[0-9]+:[0-9]+\]]], -v{{\[[0-9]+:[0-9]+\]}} +; SI-UNSAFE: v_rcp_f64_e64 [[RCP:v\[[0-9]+:[0-9]+\]]], -[[SQRT]] +; SI-UNSAFE: buffer_store_dwordx2 [[RCP]] +define void @neg_rsq_neg_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind { + %val = load double, double addrspace(1)* %in, align 4 + %val.fneg = fsub double -0.0, %val + %sqrt = call double @llvm.sqrt.f64(double %val.fneg) + %div = fdiv double -1.0, %sqrt + store double %div, double addrspace(1)* %out, align 4 + ret void +} diff --git a/test/CodeGen/AMDGPU/runtime-metadata.ll b/test/CodeGen/AMDGPU/runtime-metadata.ll index 052ad5b9c15b..abdbc325fd4d 100644 --- a/test/CodeGen/AMDGPU/runtime-metadata.ll +++ b/test/CodeGen/AMDGPU/runtime-metadata.ll @@ -1,4 +1,5 @@ -; RUN: llc -mtriple=amdgcn--amdhsa < %s | FileCheck %s +; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj -o - < %s | llvm-readobj -amdgpu-runtime-metadata | FileCheck %s +; RUN: llc -mtriple=amdgcn--amdhsa -filetype=obj -amdgpu-dump-rtmd -amdgpu-check-rtmd-parser %s -o - 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=PARSER %s %struct.A = type { i8, float } %opencl.image1d_t = type opaque @@ -9,805 +10,339 @@ %struct.B = type { i32 addrspace(1)*} %opencl.clk_event_t = type opaque -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .short 256 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 3 -; CHECK-NEXT: .short 200 - -; CHECK-LABEL:{{^}}test_char: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "test_char" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 1 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 1 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .ascii "char" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 1 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 +; CHECK: --- +; CHECK-NEXT: { amd.MDVersion: [ 2, 0 ], amd.PrintfInfo: [ '1:1:4:%d\n', '2:1:8:%g\n' ], amd.Kernels: +; CHECK-NEXT: - { amd.KernelName: test_char, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 1, amd.ArgAlign: 1, amd.ArgKind: 0, amd.ArgValueType: 1, amd.ArgTypeName: char, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_char(i8 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !9 !kernel_arg_base_type !9 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_ushort2: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 12 -; CHECK-NEXT: .ascii "test_ushort2" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 7 -; CHECK-NEXT: .ascii "ushort2" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 4 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_ushort2, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 4, amd.ArgTypeName: ushort2, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_ushort2(<2 x i16> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !10 !kernel_arg_base_type !10 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_int3: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "test_int3" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .ascii "int3" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_int3, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 16, amd.ArgAlign: 16, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_int3(<3 x i32> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !11 !kernel_arg_base_type !11 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_ulong4: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 11 -; CHECK-NEXT: .ascii "test_ulong4" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 32 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 32 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 6 -; CHECK-NEXT: .ascii "ulong4" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 10 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_ulong4, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 32, amd.ArgAlign: 32, amd.ArgKind: 0, amd.ArgValueType: 10, amd.ArgTypeName: ulong4, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_ulong4(<4 x i64> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !12 !kernel_arg_base_type !12 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_half8: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 10 -; CHECK-NEXT: .ascii "test_half8" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "half8" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 5 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_half8, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 16, amd.ArgAlign: 16, amd.ArgKind: 0, amd.ArgValueType: 5, amd.ArgTypeName: half8, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_half8(<8 x half> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !13 !kernel_arg_base_type !13 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_float16: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 12 -; CHECK-NEXT: .ascii "test_float16" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 64 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 64 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 7 -; CHECK-NEXT: .ascii "float16" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 8 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_float16, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 64, amd.ArgAlign: 64, amd.ArgKind: 0, amd.ArgValueType: 8, amd.ArgTypeName: float16, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_float16(<16 x float> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !14 !kernel_arg_base_type !14 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_double16: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 13 -; CHECK-NEXT: .ascii "test_double16" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 128 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 128 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .ascii "double16" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 11 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_double16, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 128, amd.ArgAlign: 128, amd.ArgKind: 0, amd.ArgValueType: 11, amd.ArgTypeName: double16, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_double16(<16 x double> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !15 !kernel_arg_base_type !15 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_pointer: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 12 -; CHECK-NEXT: .ascii "test_pointer" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_pointer, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_pointer(i32 addrspace(1)* %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !16 !kernel_arg_base_type !16 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_image: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 10 -; CHECK-NEXT: .ascii "test_image" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "image2d_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_image, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 4, amd.ArgValueType: 0, amd.ArgTypeName: image2d_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_image(%opencl.image2d_t addrspace(1)* %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !17 !kernel_arg_base_type !17 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_sampler: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 12 -; CHECK-NEXT: .ascii "test_sampler" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "sampler_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 3 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_sampler, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 3, amd.ArgValueType: 6, amd.ArgTypeName: sampler_t, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_sampler(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !18 !kernel_arg_base_type !18 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_queue: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 10 -; CHECK-NEXT: .ascii "test_queue" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 7 -; CHECK-NEXT: .ascii "queue_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_queue, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 6, amd.ArgValueType: 0, amd.ArgTypeName: queue_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_queue(%opencl.queue_t addrspace(1)* %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !19 !kernel_arg_base_type !19 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_struct: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 11 -; CHECK-NEXT: .ascii "test_struct" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .ascii "struct A" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_struct, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 1, amd.ArgValueType: 0, amd.ArgTypeName: struct A, amd.ArgAddrQual: 0, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_struct(%struct.A* byval %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !20 !kernel_arg_base_type !20 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_i128: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "test_i128" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .ascii "i128" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_i128, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 16, amd.ArgAlign: 8, amd.ArgKind: 0, amd.ArgValueType: 0, amd.ArgTypeName: i128, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_i128(i128 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !21 !kernel_arg_base_type !21 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_multi_arg: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 14 -; CHECK-NEXT: .ascii "test_multi_arg" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 3 -; CHECK-NEXT: .ascii "int" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 6 -; CHECK-NEXT: .ascii "short2" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 3 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "char3" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 1 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_multi_arg, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 3, amd.ArgTypeName: short2, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 1, amd.ArgTypeName: char3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_multi_arg(i32 %a, <2 x i16> %b, <3 x i8> %c) !kernel_arg_addr_space !22 !kernel_arg_access_qual !23 !kernel_arg_type !24 !kernel_arg_base_type !24 !kernel_arg_type_qual !25 { ret void } -; CHECK-LABEL:{{^}}test_addr_space: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 15 -; CHECK-NEXT: .ascii "test_addr_space" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 3 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_addr_space, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 2, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 4, amd.ArgKind: 2, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_addr_space(i32 addrspace(1)* %g, i32 addrspace(2)* %c, i32 addrspace(3)* %l) !kernel_arg_addr_space !50 !kernel_arg_access_qual !23 !kernel_arg_type !51 !kernel_arg_base_type !51 !kernel_arg_type_qual !25 { ret void } -; CHECK-LABEL:{{^}}test_type_qual: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 14 -; CHECK-NEXT: .ascii "test_type_qual" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 19 -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 17 -; CHECK-NEXT: .byte 18 -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "int *" -; CHECK-NEXT: .byte 20 -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_type_qual, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0, amd.ArgIsVolatile: 1 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0, amd.ArgIsConst: 1, amd.ArgIsRestrict: 1 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 5, amd.ArgValueType: 0, amd.ArgTypeName: 'int *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0, amd.ArgIsPipe: 1 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_type_qual(i32 addrspace(1)* %a, i32 addrspace(1)* %b, %opencl.pipe_t addrspace(1)* %c) !kernel_arg_addr_space !22 !kernel_arg_access_qual !23 !kernel_arg_type !51 !kernel_arg_base_type !51 !kernel_arg_type_qual !70 { ret void } -; CHECK-LABEL:{{^}}test_access_qual: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .ascii "test_access_qual" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "image1d_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "image2d_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 9 -; CHECK-NEXT: .ascii "image3d_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 2 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 3 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 - +; CHECK-NEXT: - { amd.KernelName: test_access_qual, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 4, amd.ArgValueType: 0, amd.ArgTypeName: image1d_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 1 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 4, amd.ArgValueType: 0, amd.ArgTypeName: image2d_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 2 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 4, amd.ArgValueType: 0, amd.ArgTypeName: image3d_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 3 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_access_qual(%opencl.image1d_t addrspace(1)* %ro, %opencl.image2d_t addrspace(1)* %wo, %opencl.image3d_t addrspace(1)* %rw) !kernel_arg_addr_space !60 !kernel_arg_access_qual !61 !kernel_arg_type !62 !kernel_arg_base_type !62 !kernel_arg_type_qual !25 { ret void } -; CHECK-LABEL:{{^}}test_reqd_wgs_vec_type_hint: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 27 -; CHECK-NEXT: .ascii "test_reqd_wgs_vec_type_hint" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 3 -; CHECK-NEXT: .ascii "int" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 21 -; CHECK-NEXT: .long 1 -; CHECK-NEXT: .long 2 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 23 -; CHECK-NEXT: .long 3 -; CHECK-NEXT: .ascii "int" -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_half, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: half, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_half(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !26 { + ret void +} -define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !5 !reqd_work_group_size !6 { +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_float, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: float, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_float(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !27 { ret void } -; CHECK-LABEL:{{^}}test_wgs_hint_vec_type_hint: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 27 -; CHECK-NEXT: .ascii "test_wgs_hint_vec_type_hint" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 3 -; CHECK-NEXT: .ascii "int" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 22 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .long 32 -; CHECK-NEXT: .byte 23 -; CHECK-NEXT: .long 5 -; CHECK-NEXT: .ascii "uint4" -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_double, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: double, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_double(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !28 { + ret void +} -define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !7 !work_group_size_hint !8 { +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_char, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: char, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_char(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !29 { ret void } -; CHECK-LABEL:{{^}}test_arg_ptr_to_ptr: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 19 -; CHECK-NEXT: .ascii "test_arg_ptr_to_ptr" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 6 -; CHECK-NEXT: .ascii "int **" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_short, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: short, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_short(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !30 { + ret void +} -define amdgpu_kernel void @test_arg_ptr_to_ptr(i32 * addrspace(1)* %a) !kernel_arg_addr_space !81 !kernel_arg_access_qual !2 !kernel_arg_type !80 !kernel_arg_base_type !80 !kernel_arg_type_qual !4 { +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_long, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: long, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_long(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !31 { ret void } -; CHECK-LABEL:{{^}}test_arg_struct_contains_ptr: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 28 -; CHECK-NEXT: .ascii "test_arg_struct_contains_ptr" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 4 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .ascii "struct B" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_vec_type_hint_unknown, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.VecTypeHint: unknown, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_vec_type_hint_unknown(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !32 { + ret void +} -define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B * byval %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !82 !kernel_arg_base_type !82 !kernel_arg_type_qual !4 { +; CHECK-NEXT: - { amd.KernelName: test_reqd_wgs_vec_type_hint, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.ReqdWorkGroupSize: [ 1, 2, 4 ], amd.VecTypeHint: int, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_reqd_wgs_vec_type_hint(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !5 !reqd_work_group_size !6 { ret void } -; CHECK-LABEL:{{^}}test_arg_vector_of_ptr: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 22 -; CHECK-NEXT: .ascii "test_arg_vector_of_ptr" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 16 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 47 -; CHECK-NEXT: .ascii "global int* __attribute__((ext_vector_type(2)))" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 6 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_wgs_hint_vec_type_hint, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.WorkGroupSizeHint: [ 8, 16, 32 ], amd.VecTypeHint: uint4, amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: int, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_wgs_hint_vec_type_hint(i32 %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !3 !kernel_arg_base_type !3 !kernel_arg_type_qual !4 !vec_type_hint !7 !work_group_size_hint !8 { + ret void +} +; CHECK-NEXT: - { amd.KernelName: test_arg_ptr_to_ptr, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 6, amd.ArgTypeName: 'int **', amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_arg_ptr_to_ptr(i32 * addrspace(1)* %a) !kernel_arg_addr_space !81 !kernel_arg_access_qual !2 !kernel_arg_type !80 !kernel_arg_base_type !80 !kernel_arg_type_qual !4 { + ret void +} +; CHECK-NEXT: - { amd.KernelName: test_arg_struct_contains_ptr, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgKind: 1, amd.ArgValueType: 0, amd.ArgTypeName: struct B, amd.ArgAddrQual: 0, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } +define amdgpu_kernel void @test_arg_struct_contains_ptr(%struct.B * byval %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !82 !kernel_arg_base_type !82 !kernel_arg_type_qual !4 { + ret void +} + +; CHECK-NEXT: - { amd.KernelName: test_arg_vector_of_ptr, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 16, amd.ArgAlign: 16, amd.ArgKind: 0, amd.ArgValueType: 6, amd.ArgTypeName: 'global int* __attribute__((ext_vector_type(2)))', amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_arg_vector_of_ptr(<2 x i32 addrspace(1)*> %a) !kernel_arg_addr_space !1 !kernel_arg_access_qual !2 !kernel_arg_type !83 !kernel_arg_base_type !83 !kernel_arg_type_qual !4 { ret void } -; CHECK-LABEL:{{^}}test_arg_unknown_builtin_type: -; CHECK: .section .AMDGPU.runtime_metadata -; CHECK-NEXT: .byte 4 -; CHECK-NEXT: .byte 6 -; CHECK-NEXT: .long 29 -; CHECK-NEXT: .ascii "test_arg_unknown_builtin_type" -; CHECK-NEXT: .byte 7 -; CHECK-NEXT: .byte 9 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 10 -; CHECK-NEXT: .long 8 -; CHECK-NEXT: .byte 11 -; CHECK-NEXT: .long 11 -; CHECK-NEXT: .ascii "clk_event_t" -; CHECK-NEXT: .byte 13 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 14 -; CHECK-NEXT: .short 0 -; CHECK-NEXT: .byte 16 -; CHECK-NEXT: .byte 0 -; CHECK-NEXT: .byte 15 -; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .byte 8 -; CHECK-NEXT: .byte 5 +; CHECK-NEXT: - { amd.KernelName: test_arg_unknown_builtin_type, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 0, amd.ArgTypeName: clk_event_t, amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } define amdgpu_kernel void @test_arg_unknown_builtin_type(%opencl.clk_event_t addrspace(1)* %a) !kernel_arg_addr_space !81 !kernel_arg_access_qual !2 !kernel_arg_type !84 !kernel_arg_base_type !84 !kernel_arg_type_qual !4 { ret void } +; CHECK-NEXT: - { amd.KernelName: test_pointee_align, amd.Language: OpenCL C, amd.LanguageVersion: [ 2, 0 ], amd.Args: +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 1, amd.ArgValueType: 9, amd.ArgTypeName: 'long *', amd.ArgAddrQual: 1, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 1, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 2, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char2 *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 4, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char3 *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 4, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char4 *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 8, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char8 *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 4, amd.ArgAlign: 4, amd.ArgPointeeAlign: 16, amd.ArgKind: 2, amd.ArgValueType: 1, amd.ArgTypeName: 'char16 *', amd.ArgAddrQual: 3, amd.ArgAccQual: 0 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 7, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 8, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 9, amd.ArgValueType: 9 } +; CHECK-NEXT: - { amd.ArgSize: 8, amd.ArgAlign: 8, amd.ArgKind: 11, amd.ArgValueType: 1, amd.ArgAddrQual: 1 } } } +define amdgpu_kernel void @test_pointee_align(i64 addrspace(1)* %a, i8 addrspace(3)* %b, <2 x i8> addrspace(3)* %c, <3 x i8> addrspace(3)* %d, <4 x i8> addrspace(3)* %e, <8 x i8> addrspace(3)* %f, <16 x i8> addrspace(3)* %g) !kernel_arg_addr_space !91 !kernel_arg_access_qual !92 !kernel_arg_type !93 !kernel_arg_base_type !93 !kernel_arg_type_qual !94 { + ret void +} + +; CHECK-NEXT:... + +; PARSER: AMDGPU runtime metadata parser test passes. + +!llvm.printf.fmts = !{!100, !101} + !1 = !{i32 0} !2 = !{!"none"} !3 = !{!"int"} @@ -833,6 +368,13 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(%opencl.clk_event_t add !23 = !{!"none", !"none", !"none"} !24 = !{!"int", !"short2", !"char3"} !25 = !{!"", !"", !""} +!26 = !{half undef, i32 1} +!27 = !{float undef, i32 1} +!28 = !{double undef, i32 1} +!29 = !{i8 undef, i32 1} +!30 = !{i16 undef, i32 1} +!31 = !{i64 undef, i32 1} +!32 = !{i32 *undef, i32 1} !50 = !{i32 1, i32 2, i32 3} !51 = !{!"int *", !"int *", !"int *"} !60 = !{i32 1, i32 1, i32 1} @@ -846,3 +388,9 @@ define amdgpu_kernel void @test_arg_unknown_builtin_type(%opencl.clk_event_t add !84 = !{!"clk_event_t"} !opencl.ocl.version = !{!90} !90 = !{i32 2, i32 0} +!91 = !{i32 0, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3} +!92 = !{!"none", !"none", !"none", !"none", !"none", !"none", !"none"} +!93 = !{!"long *", !"char *", !"char2 *", !"char3 *", !"char4 *", !"char8 *", !"char16 *"} +!94 = !{!"", !"", !"", !"", !"", !"", !""} +!100 = !{!"1:1:4:%d\5Cn"} +!101 = !{!"2:1:8:%g\5Cn"} diff --git a/test/CodeGen/AMDGPU/s_addk_i32.ll b/test/CodeGen/AMDGPU/s_addk_i32.ll index 987056010e69..bd6596a14920 100644 --- a/test/CodeGen/AMDGPU/s_addk_i32.ll +++ b/test/CodeGen/AMDGPU/s_addk_i32.ll @@ -91,3 +91,19 @@ define void @no_s_addk_i32_k0(i32 addrspace(1)* %out, i32 %b) { store i32 %add, i32 addrspace(1)* %out ret void } + +@lds = addrspace(3) global [512 x i32] undef, align 4 + +; SI-LABEL: {{^}}commute_s_addk_i32: +; SI: s_addk_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_s_addk_i32(i32 addrspace(1)* %out, i32 %b) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %add = add i32 %size, %b + call void asm sideeffect "; foo $0, $1", "v,s"([512 x i32] addrspace(3)* @lds, i32 %add) + ret void +} + +declare i32 @llvm.amdgcn.groupstaticsize() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/s_movk_i32.ll b/test/CodeGen/AMDGPU/s_movk_i32.ll index 47c7fbb6dd6a..e422270fc4ac 100644 --- a/test/CodeGen/AMDGPU/s_movk_i32.ll +++ b/test/CodeGen/AMDGPU/s_movk_i32.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; SI-LABEL: {{^}}s_movk_i32_k0: @@ -11,6 +11,7 @@ define void @s_movk_i32_k0(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 4295032831 ; ((1 << 16) - 1) | (1 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 4295032831) ret void } @@ -24,6 +25,7 @@ define void @s_movk_i32_k1(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 4295000063 ; ((1 << 15) - 1) | (1 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 4295000063) ret void } @@ -37,6 +39,7 @@ define void @s_movk_i32_k2(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 274877939711 ; ((1 << 15) - 1) | (64 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 274877939711) ret void } @@ -50,6 +53,7 @@ define void @s_movk_i32_k3(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 4295000064 ; (1 << 15) | (1 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 4295000064) ret void } @@ -63,6 +67,7 @@ define void @s_movk_i32_k4(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 4295098368 ; (1 << 17) | (1 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 4295098368) ret void } @@ -77,6 +82,7 @@ define void @s_movk_i32_k5(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 18374967954648334319 ; -17 & 0xff00ffffffffffff store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 18374967954648334319) ret void } @@ -90,6 +96,7 @@ define void @s_movk_i32_k6(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 270582939713 ; 65 | (63 << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 270582939713) ret void } @@ -104,10 +111,10 @@ define void @s_movk_i32_k7(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 70368744185856; ((1 << 13)) | ((1 << 14) << 32) store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 70368744185856) ret void } - ; SI-LABEL: {{^}}s_movk_i32_k8: ; SI-DAG: s_movk_i32 [[LO_S_IMM:s[0-9]+]], 0x8000{{$}} ; SI-DAG: s_mov_b32 [[HI_S_IMM:s[0-9]+]], 0x11111111{{$}} @@ -119,6 +126,7 @@ define void @s_movk_i32_k8(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 1229782942255906816 ; 0x11111111ffff8000 store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 1229782942255906816) ret void } @@ -133,6 +141,7 @@ define void @s_movk_i32_k9(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 add %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 1229782942255906817 ; 0x11111111ffff8001 store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 1229782942255906817) ret void } @@ -147,6 +156,7 @@ define void @s_movk_i32_k10(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 ad %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 1229782942255909000 ; 0x11111111ffff8888 store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 1229782942255909000) ret void } @@ -161,6 +171,7 @@ define void @s_movk_i32_k11(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 ad %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 1229782942255910911 ; 0x11111111ffff8fff store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 1229782942255910911) ret void } @@ -175,5 +186,6 @@ define void @s_movk_i32_k12(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 ad %loada = load i64, i64 addrspace(1)* %a, align 4 %or = or i64 %loada, 1229782942255902721 ; 0x11111111ffff7001 store i64 %or, i64 addrspace(1)* %out + call void asm sideeffect "; use $0", "s"(i64 1229782942255902721) ret void } diff --git a/test/CodeGen/AMDGPU/s_mulk_i32.ll b/test/CodeGen/AMDGPU/s_mulk_i32.ll index 33d7eeacdb83..a1462193ebb3 100644 --- a/test/CodeGen/AMDGPU/s_mulk_i32.ll +++ b/test/CodeGen/AMDGPU/s_mulk_i32.ll @@ -39,3 +39,19 @@ define void @no_s_mulk_i32_k0(i32 addrspace(1)* %out, i32 %b) { store i32 %mul, i32 addrspace(1)* %out ret void } + +@lds = addrspace(3) global [512 x i32] undef, align 4 + +; SI-LABEL: {{^}}commute_s_mulk_i32: +; SI: s_mulk_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_s_mulk_i32(i32 addrspace(1)* %out, i32 %b) #0 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %add = mul i32 %size, %b + call void asm sideeffect "; foo $0, $1", "v,s"([512 x i32] addrspace(3)* @lds, i32 %add) + ret void +} + +declare i32 @llvm.amdgcn.groupstaticsize() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/sad.ll b/test/CodeGen/AMDGPU/sad.ll new file mode 100644 index 000000000000..534483401638 --- /dev/null +++ b/test/CodeGen/AMDGPU/sad.ll @@ -0,0 +1,283 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +; GCN-LABEL: {{^}}v_sad_u32_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %b + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_constant_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, 20 +define void @v_sad_u32_constant_pat1(i32 addrspace(1)* %out, i32 %a) { + %icmp0 = icmp ugt i32 %a, 90 + %t0 = select i1 %icmp0, i32 %a, i32 90 + + %icmp1 = icmp ule i32 %a, 90 + %t1 = select i1 %icmp1, i32 %a, i32 90 + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, 20 + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_pat2: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %sub0 = sub i32 %a, %b + %sub1 = sub i32 %b, %a + %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1 + + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_sub_pat1: +; GCN: s_max_u32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: s_min_u32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +define void @v_sad_u32_multi_use_sub_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %b + + %ret0 = sub i32 %t0, %t1 + store volatile i32 %ret0, i32 *undef + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_add_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_multi_use_add_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %b + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, %c + store volatile i32 %ret, i32 *undef + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_max_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_multi_use_max_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + store volatile i32 %t0, i32 *undef + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %b + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_min_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_multi_use_min_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %b + + store volatile i32 %t1, i32 *undef + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_sub_pat2: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_multi_use_sub_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %sub0 = sub i32 %a, %b + store volatile i32 %sub0, i32 *undef + %sub1 = sub i32 %b, %a + %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1 + + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_multi_use_select_pat2: +; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: v_cmp_gt_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +define void @v_sad_u32_multi_use_select_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { + %icmp0 = icmp ugt i32 %a, %b + %sub0 = sub i32 %a, %b + %sub1 = sub i32 %b, %a + %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1 + store volatile i32 %ret0, i32 *undef + + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_vector_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_vector_pat1(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b, <4 x i32> %c) { + %icmp0 = icmp ugt <4 x i32> %a, %b + %t0 = select <4 x i1> %icmp0, <4 x i32> %a, <4 x i32> %b + + %icmp1 = icmp ule <4 x i32> %a, %b + %t1 = select <4 x i1> %icmp1, <4 x i32> %a, <4 x i32> %b + + %ret0 = sub <4 x i32> %t0, %t1 + %ret = add <4 x i32> %ret0, %c + + store <4 x i32> %ret, <4 x i32> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_vector_pat2: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_vector_pat2(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b, <4 x i32> %c) { + %icmp0 = icmp ugt <4 x i32> %a, %b + %sub0 = sub <4 x i32> %a, %b + %sub1 = sub <4 x i32> %b, %a + %ret0 = select <4 x i1> %icmp0, <4 x i32> %sub0, <4 x i32> %sub1 + + %ret = add <4 x i32> %ret0, %c + + store <4 x i32> %ret, <4 x i32> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_i16_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_i16_pat1(i16 addrspace(1)* %out, i16 %a, i16 %b, i16 %c) { + + %icmp0 = icmp ugt i16 %a, %b + %t0 = select i1 %icmp0, i16 %a, i16 %b + + %icmp1 = icmp ule i16 %a, %b + %t1 = select i1 %icmp1, i16 %a, i16 %b + + %ret0 = sub i16 %t0, %t1 + %ret = add i16 %ret0, %c + + store i16 %ret, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_i16_pat2: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_i16_pat2(i16 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b, i16 zeroext %c) { + %icmp0 = icmp ugt i16 %a, %b + %sub0 = sub i16 %a, %b + %sub1 = sub i16 %b, %a + %ret0 = select i1 %icmp0, i16 %sub0, i16 %sub1 + + %ret = add i16 %ret0, %c + + store i16 %ret, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_i8_pat1: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_i8_pat1(i8 addrspace(1)* %out, i8 %a, i8 %b, i8 %c) { + %icmp0 = icmp ugt i8 %a, %b + %t0 = select i1 %icmp0, i8 %a, i8 %b + + %icmp1 = icmp ule i8 %a, %b + %t1 = select i1 %icmp1, i8 %a, i8 %b + + %ret0 = sub i8 %t0, %t1 + %ret = add i8 %ret0, %c + + store i8 %ret, i8 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_i8_pat2: +; GCN: v_sad_u32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_i8_pat2(i8 addrspace(1)* %out, i8 zeroext %a, i8 zeroext %b, i8 zeroext %c) { + %icmp0 = icmp ugt i8 %a, %b + %sub0 = sub i8 %a, %b + %sub1 = sub i8 %b, %a + %ret0 = select i1 %icmp0, i8 %sub0, i8 %sub1 + + %ret = add i8 %ret0, %c + + store i8 %ret, i8 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat1: +; GCN: v_cmp_le_u32_e32 vcc, s{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_max_u32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: v_sub_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}} +; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_mismatched_operands_pat1(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) { + %icmp0 = icmp ugt i32 %a, %b + %t0 = select i1 %icmp0, i32 %a, i32 %b + + %icmp1 = icmp ule i32 %a, %b + %t1 = select i1 %icmp1, i32 %a, i32 %d + + %ret0 = sub i32 %t0, %t1 + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sad_u32_mismatched_operands_pat2: +; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: s_sub_i32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}} +; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, s{{[0-9]+}}, v{{[0-9]+}} +define void @v_sad_u32_mismatched_operands_pat2(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) { + %icmp0 = icmp ugt i32 %a, %b + %sub0 = sub i32 %a, %d + %sub1 = sub i32 %b, %a + %ret0 = select i1 %icmp0, i32 %sub0, i32 %sub1 + + %ret = add i32 %ret0, %c + + store i32 %ret, i32 addrspace(1)* %out + ret void +} + diff --git a/test/CodeGen/AMDGPU/salu-to-valu.ll b/test/CodeGen/AMDGPU/salu-to-valu.ll index 52f3cceac2a0..37083fbbd3c5 100644 --- a/test/CodeGen/AMDGPU/salu-to-valu.ll +++ b/test/CodeGen/AMDGPU/salu-to-valu.ll @@ -170,14 +170,12 @@ entry: ; CI. ; GCN-LABEL: {{^}}smrd_valu_ci_offset_x8: +; GCN-NOHSA: s_mov_b32 [[OFFSET1:s[0-9]+]], 0x9a50{{$}} ; GCN-NOHSA-NOT: v_add ; GCN-NOHSA: s_mov_b32 [[OFFSET0:s[0-9]+]], 0x9a40{{$}} ; GCN-NOHSA-NOT: v_add -; GCN-NOHSA: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET0]] addr64{{$}} -; GCN-NOHSA-NOT: v_add -; GCN-NOHSA: s_mov_b32 [[OFFSET1:s[0-9]+]], 0x9a50{{$}} -; GCN-NOHSA-NOT: v_add ; GCN-NOHSA: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET1]] addr64{{$}} +; GCN-NOHSA: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, s[{{[0-9]+:[0-9]+}}], [[OFFSET0]] addr64{{$}} ; GCN-NOHSA: v_or_b32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}} ; GCN-NOHSA: v_or_b32_e32 {{v[0-9]+}}, {{s[0-9]+}}, {{v[0-9]+}} @@ -459,5 +457,51 @@ bb7: ; preds = %bb3 ret void } +; GCN-LABEL: {{^}}phi_visit_order: +; GCN: v_add_i32_e32 v{{[0-9]+}}, vcc, 1, v{{[0-9]+}} +define void @phi_visit_order() { +bb: + br label %bb1 + +bb1: + %tmp = phi i32 [ 0, %bb ], [ %tmp5, %bb4 ] + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %cnd = icmp eq i32 %tid, 0 + br i1 %cnd, label %bb4, label %bb2 + +bb2: + %tmp3 = add nsw i32 %tmp, 1 + br label %bb4 + +bb4: + %tmp5 = phi i32 [ %tmp3, %bb2 ], [ %tmp, %bb1 ] + br label %bb1 +} + +; GCN-LABEL: {{^}}phi_imm_in_sgprs +; GCN: s_movk_i32 [[A:s[0-9]+]], 0x400 +; GCN: s_movk_i32 [[B:s[0-9]+]], 0x400 +; GCN: [[LOOP_LABEL:[0-9a-zA-Z_]+]]: +; GCN: s_xor_b32 [[B]], [[B]], [[A]] +; GCN: s_cbranch_scc{{[01]}} [[LOOP_LABEL]] +define void @phi_imm_in_sgprs(i32 addrspace(3)* %out, i32 %cond) { +entry: + br label %loop + +loop: + %i = phi i32 [0, %entry], [%i.add, %loop] + %offset = phi i32 [1024, %entry], [%offset.xor, %loop] + %offset.xor = xor i32 %offset, 1024 + %offset.i = add i32 %offset.xor, %i + %ptr = getelementptr i32, i32 addrspace(3)* %out, i32 %offset.i + store i32 0, i32 addrspace(3)* %ptr + %i.add = add i32 %i, 1 + %cmp = icmp ult i32 %i.add, %cond + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + attributes #0 = { nounwind readnone } attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/scalar-store-cache-flush.mir b/test/CodeGen/AMDGPU/scalar-store-cache-flush.mir new file mode 100644 index 000000000000..af71086e542f --- /dev/null +++ b/test/CodeGen/AMDGPU/scalar-store-cache-flush.mir @@ -0,0 +1,173 @@ +# RUN: llc -march=amdgcn -run-pass si-insert-waits %s -o - | FileCheck %s + +--- | + define void @basic_insert_dcache_wb() { + ret void + } + + define void @explicit_flush_after() { + ret void + } + + define void @explicit_flush_before() { + ret void + } + + define void @no_scalar_store() { + ret void + } + + define void @multi_block_store() { + bb0: + br i1 undef, label %bb1, label %bb2 + + bb1: + ret void + + bb2: + ret void + } + + define void @one_block_store() { + bb0: + br i1 undef, label %bb1, label %bb2 + + bb1: + ret void + + bb2: + ret void + } + + define amdgpu_ps float @si_return() { + ret float undef + } + +... +--- +# CHECK-LABEL: name: basic_insert_dcache_wb +# CHECK: bb.0: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +name: basic_insert_dcache_wb +tracksRegLiveness: false + +body: | + bb.0: + S_STORE_DWORD_SGPR undef %sgpr2, undef %sgpr0_sgpr1, undef %m0, 0 + S_ENDPGM +... +--- +# Already has an explicitly requested flush after the last store. +# CHECK-LABEL: name: explicit_flush_after +# CHECK: bb.0: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +name: explicit_flush_after +tracksRegLiveness: false + +body: | + bb.0: + S_STORE_DWORD_SGPR undef %sgpr2, undef %sgpr0_sgpr1, undef %m0, 0 + S_DCACHE_WB + S_ENDPGM +... +--- +# Already has an explicitly requested flush before the last store. +# CHECK-LABEL: name: explicit_flush_before +# CHECK: bb.0: +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +name: explicit_flush_before +tracksRegLiveness: false + +body: | + bb.0: + S_DCACHE_WB + S_STORE_DWORD_SGPR undef %sgpr2, undef %sgpr0_sgpr1, undef %m0, 0 + S_ENDPGM +... +--- +# CHECK-LABEL: no_scalar_store +# CHECK: bb.0 +# CHECK-NEXT: S_ENDPGM +name: no_scalar_store +tracksRegLiveness: false + +body: | + bb.0: + S_ENDPGM +... + +# CHECK-LABEL: name: multi_block_store +# CHECK: bb.0: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +# CHECK: bb.1: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +name: multi_block_store +tracksRegLiveness: false + +body: | + bb.0: + S_STORE_DWORD_SGPR undef %sgpr2, undef %sgpr0_sgpr1, undef %m0, 0 + S_ENDPGM + + bb.1: + S_STORE_DWORD_SGPR undef %sgpr4, undef %sgpr6_sgpr7, undef %m0, 0 + S_ENDPGM +... +... + +# This one should be able to omit the flush in the storeless block but +# this isn't handled now. + +# CHECK-LABEL: name: one_block_store +# CHECK: bb.0: +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +# CHECK: bb.1: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: S_ENDPGM + +name: one_block_store +tracksRegLiveness: false + +body: | + bb.0: + S_ENDPGM + + bb.1: + S_STORE_DWORD_SGPR undef %sgpr4, undef %sgpr6_sgpr7, undef %m0, 0 + S_ENDPGM +... +--- +# CHECK-LABEL: name: si_return +# CHECK: bb.0: +# CHECK-NEXT: S_STORE_DWORD +# CHECK-NEXT: S_WAITCNT +# CHECK-NEXT: S_DCACHE_WB +# CHECK-NEXT: SI_RETURN + +name: si_return +tracksRegLiveness: false + +body: | + bb.0: + S_STORE_DWORD_SGPR undef %sgpr2, undef %sgpr0_sgpr1, undef %m0, 0 + SI_RETURN undef %vgpr0 +... diff --git a/test/CodeGen/AMDGPU/schedule-global-loads.ll b/test/CodeGen/AMDGPU/schedule-global-loads.ll index 1bf109dec032..32c456bd2ceb 100644 --- a/test/CodeGen/AMDGPU/schedule-global-loads.ll +++ b/test/CodeGen/AMDGPU/schedule-global-loads.ll @@ -1,6 +1,5 @@ ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=FUNC -check-prefix=SI %s - ; FIXME: This currently doesn't do a great job of clustering the ; loads, which end up with extra moves between them. Right now, it ; seems the only things areLoadsFromSameBasePtr is accomplishing is diff --git a/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll b/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll new file mode 100644 index 000000000000..c0e8e58556fc --- /dev/null +++ b/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll @@ -0,0 +1,55 @@ +; RUN: llc -march=amdgcn < %s | FileCheck %s +; REQUIRES: asserts +; +; This test used to crash with the following assertion: +; llc: include/llvm/ADT/IntervalMap.h:632: unsigned int llvm::IntervalMapImpl::LeafNode<llvm::SlotIndex, llvm::LiveInterval *, 8, llvm::IntervalMapInfo<llvm::SlotIndex> >::insertFrom(unsigned int &, unsigned int, KeyT, KeyT, ValT) [KeyT = llvm::SlotIndex, ValT = llvm::LiveInterval *, N = 8, Traits = llvm::IntervalMapInfo<llvm::SlotIndex>]: Assertion `(i == Size || Traits::stopLess(b, start(i))) && "Overlapping insert"' failed. +; +; This was related to incorrectly calculating subregister live ranges +; (i.e. live interval subranges): subregister defs are not uses for that +; purpose. +; +; Check for a valid output: +; CHECK: tbuffer_store_format_x + +target triple = "amdgcn--" + +define amdgpu_gs void @main(i32 inreg %arg) #0 { +main_body: + %tmp = call float @llvm.SI.load.const(<16 x i8> undef, i32 20) + %tmp1 = call float @llvm.SI.load.const(<16 x i8> undef, i32 24) + %tmp2 = call float @llvm.SI.load.const(<16 x i8> undef, i32 48) + %array_vector3 = insertelement <4 x float> zeroinitializer, float %tmp2, i32 3 + %array_vector5 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1 + %array_vector6 = insertelement <4 x float> %array_vector5, float undef, i32 2 + %array_vector9 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp1, i32 1 + %array_vector10 = insertelement <4 x float> %array_vector9, float 0.000000e+00, i32 2 + %array_vector11 = insertelement <4 x float> %array_vector10, float undef, i32 3 + %tmp3 = call i32 @llvm.SI.buffer.load.dword.i32.i32(<16 x i8> undef, i32 undef, i32 4864, i32 0, i32 1, i32 0, i32 1, i32 0, i32 0) + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 %tmp3, i32 1, i32 36, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + %bc = bitcast <4 x float> %array_vector3 to <4 x i32> + %tmp4 = extractelement <4 x i32> %bc, i32 undef + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 %tmp4, i32 1, i32 48, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + %bc49 = bitcast <4 x float> %array_vector11 to <4 x i32> + %tmp5 = extractelement <4 x i32> %bc49, i32 undef + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 %tmp5, i32 1, i32 72, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + %array_vector21 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1 + %array_vector22 = insertelement <4 x float> %array_vector21, float undef, i32 2 + %array_vector23 = insertelement <4 x float> %array_vector22, float undef, i32 3 + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 undef, i32 1, i32 28, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + %bc52 = bitcast <4 x float> %array_vector23 to <4 x i32> + %tmp6 = extractelement <4 x i32> %bc52, i32 undef + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 %tmp6, i32 1, i32 64, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 undef, i32 1, i32 20, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 undef, i32 1, i32 56, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + call void @llvm.SI.tbuffer.store.i32(<16 x i8> undef, i32 undef, i32 1, i32 92, i32 %arg, i32 0, i32 4, i32 4, i32 1, i32 0, i32 1, i32 1, i32 0) + ret void +} + +declare float @llvm.SI.load.const(<16 x i8>, i32) #1 +declare i32 @llvm.SI.buffer.load.dword.i32.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32) #2 +declare void @llvm.SI.tbuffer.store.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) #3 + +attributes #0 = { nounwind "target-cpu"="tonga" } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind readonly } +attributes #3 = { nounwind } diff --git a/test/CodeGen/AMDGPU/scratch-buffer.ll b/test/CodeGen/AMDGPU/scratch-buffer.ll index a66f074123c1..94101f0b92b6 100644 --- a/test/CodeGen/AMDGPU/scratch-buffer.ll +++ b/test/CodeGen/AMDGPU/scratch-buffer.ll @@ -9,9 +9,8 @@ ; should be able to reuse the same regiser for each scratch buffer access. ; GCN-LABEL: {{^}}legal_offset_fi: -; GCN: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0{{$}} -; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen -; GCN: v_mov_b32_e32 [[OFFSET]], 0x8000 +; GCN: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+$}} +; GCN: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0x8000 ; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}} define void @legal_offset_fi(i32 addrspace(1)* %out, i32 %cond, i32 %if_offset, i32 %else_offset) { @@ -48,8 +47,10 @@ done: } ; GCN-LABEL: {{^}}legal_offset_fi_offset: -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}} -; GCN: v_add_i32_e32 [[OFFSET:v[0-9]+]], vcc, 0x8000 +; GCN-DAG: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}} +; This constant isn't folded, because it has multiple uses. +; GCN-DAG: v_mov_b32_e32 [[K8000:v[0-9]+]], 0x8000 +; GCN-DAG: v_add_i32_e32 [[OFFSET:v[0-9]+]], vcc, [[K8000]] ; GCN: buffer_store_dword v{{[0-9]+}}, [[OFFSET]], s[{{[0-9]+}}:{{[0-9]+}}], s{{[0-9]+}} offen{{$}} define void @legal_offset_fi_offset(i32 addrspace(1)* %out, i32 %cond, i32 addrspace(1)* %offsets, i32 %if_offset, i32 %else_offset) { @@ -97,7 +98,7 @@ entry: } ; GCN-LABEL: {{^}}pos_vaddr_offset: -; GCN: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offen offset:16 +; GCN: buffer_store_dword v{{[0-9]+}}, off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:16 define void @pos_vaddr_offset(i32 addrspace(1)* %out, i32 %offset) { entry: %array = alloca [8192 x i32] diff --git a/test/CodeGen/AMDGPU/sdiv.ll b/test/CodeGen/AMDGPU/sdiv.ll index 29d893414c07..66caad4677b6 100644 --- a/test/CodeGen/AMDGPU/sdiv.ll +++ b/test/CodeGen/AMDGPU/sdiv.ll @@ -156,3 +156,16 @@ define void @v_sdiv_i25(i32 addrspace(1)* %out, i25 addrspace(1)* %in) { ; store i64 %result, i64 addrspace(1)* %out, align 8 ; ret void ; } + +; FUNC-LABEL: @scalarize_mulhs_4xi32 +; SI: v_mul_hi_i32 +; SI: v_mul_hi_i32 +; SI: v_mul_hi_i32 +; SI: v_mul_hi_i32 + +define void @scalarize_mulhs_4xi32(<4 x i32> addrspace(1)* nocapture readonly %in, <4 x i32> addrspace(1)* nocapture %out) { + %1 = load <4 x i32>, <4 x i32> addrspace(1)* %in, align 16 + %2 = sdiv <4 x i32> %1, <i32 53668, i32 53668, i32 53668, i32 53668> + store <4 x i32> %2, <4 x i32> addrspace(1)* %out, align 16 + ret void +} diff --git a/test/CodeGen/AMDGPU/select-i1.ll b/test/CodeGen/AMDGPU/select-i1.ll index 2406831b94c5..adc054b4895b 100644 --- a/test/CodeGen/AMDGPU/select-i1.ll +++ b/test/CodeGen/AMDGPU/select-i1.ll @@ -17,7 +17,7 @@ define void @select_i1(i1 addrspace(1)* %out, i32 %cond, i1 %a, i1 %b) nounwind ; SI-DAG: buffer_load_ubyte [[COND:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:44 ; SI-DAG: buffer_load_ubyte [[A:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:45 ; SI-DAG: buffer_load_ubyte [[B:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:46 -; SI: v_cmp_eq_i32_e32 vcc, 1, [[COND]] +; SI: v_cmp_eq_u32_e32 vcc, 1, [[COND]] ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, [[B]], [[A]] define void @s_minmax_i1(i1 addrspace(1)* %out, i1 zeroext %cond, i1 zeroext %a, i1 zeroext %b) nounwind { %cmp = icmp slt i1 %cond, false diff --git a/test/CodeGen/AMDGPU/select-vectors.ll b/test/CodeGen/AMDGPU/select-vectors.ll index faf8d8a12c25..240235138c8d 100644 --- a/test/CodeGen/AMDGPU/select-vectors.ll +++ b/test/CodeGen/AMDGPU/select-vectors.ll @@ -58,7 +58,7 @@ define void @s_select_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32 ; FUNC-LABEL: {{^}}v_select_v4i32: ; SI: buffer_load_dwordx4 -; SI: v_cmp_gt_u32_e64 vcc, 32, s{{[0-9]+}} +; SI: v_cmp_lt_u32_e64 vcc, s{{[0-9]+}}, 32 ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} @@ -93,13 +93,13 @@ define void @select_v8i32(<8 x i32> addrspace(1)* %out, <8 x i32> %a, <8 x i32> ; SI-DAG: s_load_dwordx2 s{{\[}}[[ALO:[0-9]+]]:[[AHI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} ; SI-DAG: s_load_dwordx2 s{{\[}}[[BLO:[0-9]+]]:[[BHI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xd|0x34}} -; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[ALO]] ; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[AHI]] -; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[BLO]] ; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[BHI]] -; SI-DAG: v_cmp_eq_i32_e64 vcc, 0, s{{[0-9]+}} +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[ALO]] +; SI-DAG: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}} ; SI: v_cndmask_b32_e32 +; SI: v_mov_b32_e32 v{{[0-9]+}}, s[[BLO]] ; SI: v_cndmask_b32_e32 ; SI: buffer_store_dwordx2 define void @s_select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b, i32 %c) nounwind { @@ -112,7 +112,7 @@ define void @s_select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x ; FUNC-LABEL: {{^}}s_select_v4f32: ; SI: s_load_dwordx4 ; SI: s_load_dwordx4 -; SI: v_cmp_eq_i32_e64 vcc, 0, s{{[0-9]+}} +; SI: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0{{$}} ; SI: v_cndmask_b32_e32 ; SI: v_cndmask_b32_e32 @@ -129,7 +129,7 @@ define void @s_select_v4f32(<4 x float> addrspace(1)* %out, <4 x float> %a, <4 x ; FUNC-LABEL: {{^}}v_select_v4f32: ; SI: buffer_load_dwordx4 -; SI: v_cmp_gt_u32_e64 vcc, 32, s{{[0-9]+}} +; SI: v_cmp_lt_u32_e64 vcc, s{{[0-9]+}}, 32 ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} ; SI: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}} diff --git a/test/CodeGen/AMDGPU/select.f16.ll b/test/CodeGen/AMDGPU/select.f16.ll new file mode 100644 index 000000000000..84084c912543 --- /dev/null +++ b/test/CodeGen/AMDGPU/select.f16.ll @@ -0,0 +1,326 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}select_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[D_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]] +; SI: v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_cmp_lt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; VI: v_cndmask_b32_e32 v[[R_F16:[0-9]+]], v[[D_F16]], v[[C_F16]], vcc +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @select_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c, + half addrspace(1)* %d) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %d.val = load half, half addrspace(1)* %d + %fcmp = fcmp olt half %a.val, %b.val + %r.val = select i1 %fcmp, half %c.val, half %d.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_f16_imm_a +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[D_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cmp_gt_f32_e32 vcc, v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]] +; SI: v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_cmp_lt_f16_e32 vcc, 0.5, v[[B_F16]] +; VI: v_cndmask_b32_e32 v[[R_F16:[0-9]+]], v[[D_F16]], v[[C_F16]], vcc +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @select_f16_imm_a( + half addrspace(1)* %r, + half addrspace(1)* %b, + half addrspace(1)* %c, + half addrspace(1)* %d) { +entry: + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %d.val = load half, half addrspace(1)* %d + %fcmp = fcmp olt half 0xH3800, %b.val + %r.val = select i1 %fcmp, half %c.val, half %d.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_f16_imm_b +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[D_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]] +; SI: v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_cmp_gt_f16_e32 vcc, 0.5, v[[A_F16]] +; VI: v_cndmask_b32_e32 v[[R_F16:[0-9]+]], v[[D_F16]], v[[C_F16]], vcc +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @select_f16_imm_b( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %c, + half addrspace(1)* %d) { +entry: + %a.val = load half, half addrspace(1)* %a + %c.val = load half, half addrspace(1)* %c + %d.val = load half, half addrspace(1)* %d + %fcmp = fcmp olt half %a.val, 0xH3800 + %r.val = select i1 %fcmp, half %c.val, half %d.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_f16_imm_c +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[D_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], v[[D_F16]] +; SI: v_cmp_nlt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; SI: v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[C_F32]], v[[D_F32]], vcc +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_cmp_nlt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; VI: v_mov_b32_e32 v[[C_F16:[0-9]+]], 0x3800{{$}} +; VI: v_cndmask_b32_e32 v[[R_F16:[0-9]+]], v[[C_F16]], v[[D_F16]], vcc +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @select_f16_imm_c( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %d) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %d.val = load half, half addrspace(1)* %d + %fcmp = fcmp olt half %a.val, %b.val + %r.val = select i1 %fcmp, half 0xH3800, half %d.val + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_f16_imm_d +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[D_F32:[0-9]+]], 0x3800{{$}} +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_cmp_lt_f32_e32 vcc, v[[A_F32]], v[[B_F32]] +; SI: v_cndmask_b32_e32 v[[R_F32:[0-9]+]], v[[D_F32]], v[[C_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]] +; VI: v_cmp_lt_f16_e32 vcc, v[[A_F16]], v[[B_F16]] +; VI: v_mov_b32_e32 v[[D_F16:[0-9]+]], 0x3800{{$}} +; VI: v_cndmask_b32_e32 v[[R_F16:[0-9]+]], v[[D_F16]], v[[C_F16]], vcc +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @select_f16_imm_d( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %fcmp = fcmp olt half %a.val, %b.val + %r.val = select i1 %fcmp, half %c.val, half 0xH3800 + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_v2f16 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_lt_f32_e64 +; SI: v_cmp_lt_f32_e32 +; VI: v_cmp_lt_f16_e32 +; VI: v_cmp_lt_f16_e64 +; GCN: v_cndmask_b32_e32 +; GCN: v_cndmask_b32_e64 +; SI: v_cvt_f16_f32_e32 +; SI: v_cvt_f16_f32_e32 +; GCN: s_endpgm +define void @select_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c, + <2 x half> addrspace(1)* %d) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %d.val = load <2 x half>, <2 x half> addrspace(1)* %d + %fcmp = fcmp olt <2 x half> %a.val, %b.val + %r.val = select <2 x i1> %fcmp, <2 x half> %c.val, <2 x half> %d.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_v2f16_imm_a +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_gt_f32_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_gt_f32_e64 +; VI: v_cmp_lt_f16_e32 +; VI: v_cmp_lt_f16_e64 +; GCN: v_cndmask_b32_e32 +; SI: v_cvt_f16_f32_e32 +; GCN: v_cndmask_b32_e64 +; SI: v_cvt_f16_f32_e32 +; GCN: s_endpgm +define void @select_v2f16_imm_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c, + <2 x half> addrspace(1)* %d) { +entry: + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %d.val = load <2 x half>, <2 x half> addrspace(1)* %d + %fcmp = fcmp olt <2 x half> <half 0xH3800, half 0xH3900>, %b.val + %r.val = select <2 x i1> %fcmp, <2 x half> %c.val, <2 x half> %d.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_v2f16_imm_b +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_lt_f32_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_lt_f32_e64 +; VI: v_cmp_gt_f16_e32 +; VI: v_cmp_gt_f16_e64 +; GCN: v_cndmask_b32_e32 +; SI: v_cvt_f16_f32_e32 +; GCN: v_cndmask_b32_e64 +; SI: v_cvt_f16_f32_e32 +; GCN: s_endpgm +define void @select_v2f16_imm_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %c, + <2 x half> addrspace(1)* %d) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %d.val = load <2 x half>, <2 x half> addrspace(1)* %d + %fcmp = fcmp olt <2 x half> %a.val, <half 0xH3800, half 0xH3900> + %r.val = select <2 x i1> %fcmp, <2 x half> %c.val, <2 x half> %d.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_v2f16_imm_c +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 + +; SI: v_cmp_lt_f32_e32 +; SI: v_cmp_lt_f32_e64 +; SI: v_cndmask_b32_e32 +; SI: v_cndmask_b32_e64 + +; VI: v_cmp_nlt_f16_e32 +; VI: v_cndmask_b32_e32 + +; VI: v_cmp_nlt_f16_e32 +; VI: v_cndmask_b32_e32 + +; SI: v_cvt_f16_f32_e32 +; SI: v_cvt_f16_f32_e32 +; GCN: s_endpgm +define void @select_v2f16_imm_c( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %d) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %d.val = load <2 x half>, <2 x half> addrspace(1)* %d + %fcmp = fcmp olt <2 x half> %a.val, %b.val + %r.val = select <2 x i1> %fcmp, <2 x half> <half 0xH3800, half 0xH3900>, <2 x half> %d.val + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}select_v2f16_imm_d +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cvt_f32_f16_e32 +; SI: v_cmp_lt_f32_e32 +; SI: v_cmp_lt_f32_e64 +; VI: v_cmp_lt_f16_e32 +; VI: v_cmp_lt_f16_e64 +; GCN: v_cndmask_b32_e32 +; GCN: v_cndmask_b32_e64 +; SI: v_cvt_f16_f32_e32 +; SI: v_cvt_f16_f32_e32 +; GCN: s_endpgm +define void @select_v2f16_imm_d( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %fcmp = fcmp olt <2 x half> %a.val, %b.val + %r.val = select <2 x i1> %fcmp, <2 x half> %c.val, <2 x half> <half 0xH3800, half 0xH3900> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} diff --git a/test/CodeGen/AMDGPU/selectcc-opt.ll b/test/CodeGen/AMDGPU/selectcc-opt.ll index 65be4a626a18..25670afec247 100644 --- a/test/CodeGen/AMDGPU/selectcc-opt.ll +++ b/test/CodeGen/AMDGPU/selectcc-opt.ll @@ -68,7 +68,7 @@ entry: } ; FUNC-LABEL: {{^}}selectcc_bool: -; SI: v_cmp_ne_i32 +; SI: v_cmp_ne_u32 ; SI-NEXT: v_cndmask_b32_e64 ; SI-NOT: cmp ; SI-NOT: cndmask diff --git a/test/CodeGen/AMDGPU/selectcc.ll b/test/CodeGen/AMDGPU/selectcc.ll index f378e15dd763..446d4ab344b2 100644 --- a/test/CodeGen/AMDGPU/selectcc.ll +++ b/test/CodeGen/AMDGPU/selectcc.ll @@ -8,7 +8,7 @@ ; EG: OR_INT ; EG: CNDE_INT ; EG: CNDE_INT -; SI: v_cmp_eq_i64 +; SI: v_cmp_eq_u64 ; SI: v_cndmask ; SI: v_cndmask define void @selectcc_i64(i64 addrspace(1) * %out, i64 %lhs, i64 %rhs, i64 %true, i64 %false) { diff --git a/test/CodeGen/AMDGPU/setcc-opt.ll b/test/CodeGen/AMDGPU/setcc-opt.ll index d2c57a810c2c..e5af10e95228 100644 --- a/test/CodeGen/AMDGPU/setcc-opt.ll +++ b/test/CodeGen/AMDGPU/setcc-opt.ll @@ -4,7 +4,7 @@ ; FUNC-LABEL: {{^}}sext_bool_icmp_eq_0: ; GCN-NOT: v_cmp -; GCN: v_cmp_ne_i32_e32 vcc, +; GCN: v_cmp_ne_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT:buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -21,7 +21,7 @@ define void @sext_bool_icmp_eq_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; FUNC-LABEL: {{^}}sext_bool_icmp_ne_0: ; GCN-NOT: v_cmp -; GCN: v_cmp_ne_i32_e32 vcc, +; GCN: v_cmp_ne_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -38,7 +38,7 @@ define void @sext_bool_icmp_ne_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; FUNC-LABEL: {{^}}sext_bool_icmp_eq_neg1: ; GCN-NOT: v_cmp -; GCN: v_cmp_eq_i32_e32 vcc, +; GCN: v_cmp_eq_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -52,7 +52,7 @@ define void @sext_bool_icmp_eq_neg1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounw ; FUNC-LABEL: {{^}}sext_bool_icmp_ne_neg1: ; GCN-NOT: v_cmp -; GCN: v_cmp_eq_i32_e32 vcc, +; GCN: v_cmp_eq_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -66,7 +66,7 @@ define void @sext_bool_icmp_ne_neg1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounw ; FUNC-LABEL: {{^}}zext_bool_icmp_eq_0: ; GCN-NOT: v_cmp -; GCN: v_cmp_ne_i32_e32 vcc, +; GCN: v_cmp_ne_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -80,7 +80,7 @@ define void @zext_bool_icmp_eq_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; FUNC-LABEL: {{^}}zext_bool_icmp_ne_0: ; GCN-NOT: v_cmp -; GCN: v_cmp_ne_i32_e32 vcc, +; GCN: v_cmp_ne_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -94,7 +94,7 @@ define void @zext_bool_icmp_ne_0(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; FUNC-LABEL: {{^}}zext_bool_icmp_eq_1: ; GCN-NOT: v_cmp -; GCN: v_cmp_eq_i32_e32 vcc, +; GCN: v_cmp_eq_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN-NEXT: s_endpgm @@ -108,7 +108,7 @@ define void @zext_bool_icmp_eq_1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind ; FUNC-LABEL: {{^}}zext_bool_icmp_ne_1: ; GCN-NOT: v_cmp -; GCN: v_cmp_eq_i32_e32 vcc, +; GCN: v_cmp_eq_u32_e32 vcc, ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN-NEXT: buffer_store_byte [[RESULT]] define void @zext_bool_icmp_ne_1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounwind { @@ -149,9 +149,13 @@ define void @zext_bool_icmp_ne_neg1(i1 addrspace(1)* %out, i32 %a, i32 %b) nounw ; SI: s_load_dword [[VALUE:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb ; VI: s_load_dword [[VALUE:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c ; GCN: s_movk_i32 [[K255:s[0-9]+]], 0xff -; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VALUE]], [[K255]] ; GCN-DAG: v_mov_b32_e32 [[VK255:v[0-9]+]], [[K255]] -; GCN: v_cmp_ne_i32_e32 vcc, [[B]], [[VK255]] +; SI-DAG: s_and_b32 [[B:s[0-9]+]], [[VALUE]], [[K255]] +; SI: v_cmp_ne_u32_e32 vcc, [[B]], [[VK255]] + +; VI-DAG: v_and_b32_e32 [[B:v[0-9]+]], [[VALUE]], [[VK255]] +; VI: v_cmp_ne_u16_e32 vcc, [[K255]], [[B]] + ; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN: buffer_store_byte [[RESULT]] ; GCN: s_endpgm @@ -164,7 +168,7 @@ define void @cmp_zext_k_i8max(i1 addrspace(1)* %out, i8 %b) nounwind { ; FUNC-LABEL: {{^}}cmp_sext_k_neg1: ; GCN: buffer_load_sbyte [[B:v[0-9]+]] -; GCN: v_cmp_ne_i32_e32 vcc, -1, [[B]]{{$}} +; GCN: v_cmp_ne_u32_e32 vcc, -1, [[B]]{{$}} ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN: buffer_store_byte [[RESULT]] ; GCN: s_endpgm @@ -178,7 +182,7 @@ define void @cmp_sext_k_neg1(i1 addrspace(1)* %out, i8 addrspace(1)* %b.ptr) nou ; FUNC-LABEL: {{^}}cmp_sext_k_neg1_i8_sext_arg: ; GCN: s_load_dword [[B:s[0-9]+]] -; GCN: v_cmp_ne_i32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], -1, [[B]] +; GCN: v_cmp_ne_u32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], [[B]], -1{{$}} ; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CMP]] ; GCN-NEXT: buffer_store_byte [[RESULT]] ; GCN: s_endpgm @@ -199,7 +203,7 @@ define void @cmp_sext_k_neg1_i8_sext_arg(i1 addrspace(1)* %out, i8 signext %b) n ; GCN: s_movk_i32 [[K:s[0-9]+]], 0xff ; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VAL]], [[K]] ; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], [[K]] -; GCN: v_cmp_ne_i32_e32 vcc, [[B]], [[VK]]{{$}} +; GCN: v_cmp_ne_u32_e32 vcc, [[B]], [[VK]]{{$}} ; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc ; GCN: buffer_store_byte [[RESULT]] ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/setcc.ll b/test/CodeGen/AMDGPU/setcc.ll index c89e712e4cb0..10d04bab9f6b 100644 --- a/test/CodeGen/AMDGPU/setcc.ll +++ b/test/CodeGen/AMDGPU/setcc.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -march=amdgcn -verify-machineinstrs | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=R600 --check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 -check-prefix=FUNC %s declare i32 @llvm.r600.read.tidig.x() nounwind readnone @@ -7,7 +7,9 @@ declare i32 @llvm.r600.read.tidig.x() nounwind readnone ; R600-DAG: SETE_INT * T{{[0-9]+\.[XYZW]}}, KC0[3].X, KC0[3].Z ; R600-DAG: SETE_INT * T{{[0-9]+\.[XYZW]}}, KC0[2].W, KC0[3].Y -define void @setcc_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) { +; GCN-DAG: v_cmp_eq_u32_e32 +; GCN-DAG: v_cmp_eq_u32_e64 +define void @setcc_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) #0 { %result = icmp eq <2 x i32> %a, %b %sext = sext <2 x i1> %result to <2 x i32> store <2 x i32> %sext, <2 x i32> addrspace(1)* %out @@ -20,10 +22,14 @@ define void @setcc_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> % ; R600-DAG: SETE_INT * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} ; R600-DAG: SETE_INT * T{{[0-9]+\.[XYZW], T[0-9]+\.[XYZW], T[0-9]+\.[XYZW]}} -define void @setcc_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { +; GCN: v_cmp_eq_u32_e32 +; GCN: v_cmp_eq_u32_e64 +; GCN: v_cmp_eq_u32_e64 +; GCN: v_cmp_eq_u32_e64 +define void @setcc_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) #0 { %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1 - %a = load <4 x i32>, <4 x i32> addrspace(1) * %in - %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr + %a = load <4 x i32>, <4 x i32> addrspace(1)* %in + %b = load <4 x i32>, <4 x i32> addrspace(1)* %b_ptr %result = icmp eq <4 x i32> %a, %b %sext = sext <4 x i1> %result to <4 x i32> store <4 x i32> %sext, <4 x i32> addrspace(1)* %out @@ -36,8 +42,8 @@ define void @setcc_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* % ; FUNC-LABEL: {{^}}f32_oeq: ; R600: SETE_DX10 -; SI: v_cmp_eq_f32 -define void @f32_oeq(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_eq_f32 +define void @f32_oeq(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp oeq float %a, %b %1 = sext i1 %0 to i32 @@ -47,8 +53,8 @@ entry: ; FUNC-LABEL: {{^}}f32_ogt: ; R600: SETGT_DX10 -; SI: v_cmp_gt_f32 -define void @f32_ogt(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_gt_f32 +define void @f32_ogt(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ogt float %a, %b %1 = sext i1 %0 to i32 @@ -58,8 +64,8 @@ entry: ; FUNC-LABEL: {{^}}f32_oge: ; R600: SETGE_DX10 -; SI: v_cmp_ge_f32 -define void @f32_oge(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_ge_f32 +define void @f32_oge(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp oge float %a, %b %1 = sext i1 %0 to i32 @@ -69,8 +75,8 @@ entry: ; FUNC-LABEL: {{^}}f32_olt: ; R600: SETGT_DX10 -; SI: v_cmp_lt_f32 -define void @f32_olt(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_lt_f32 +define void @f32_olt(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp olt float %a, %b %1 = sext i1 %0 to i32 @@ -80,8 +86,8 @@ entry: ; FUNC-LABEL: {{^}}f32_ole: ; R600: SETGE_DX10 -; SI: v_cmp_le_f32 -define void @f32_ole(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_le_f32 +define void @f32_ole(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ole float %a, %b %1 = sext i1 %0 to i32 @@ -97,9 +103,9 @@ entry: ; R600-DAG: AND_INT ; R600-DAG: SETNE_INT -; SI: v_cmp_lg_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_one(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_lg_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_one(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp one float %a, %b %1 = sext i1 %0 to i32 @@ -112,8 +118,8 @@ entry: ; R600-DAG: SETE_DX10 ; R600-DAG: AND_INT ; R600-DAG: SETNE_INT -; SI: v_cmp_o_f32 -define void @f32_ord(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_o_f32 +define void @f32_ord(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ord float %a, %b %1 = sext i1 %0 to i32 @@ -129,9 +135,9 @@ entry: ; R600-DAG: OR_INT ; R600-DAG: SETNE_INT -; SI: v_cmp_nlg_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_ueq(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_nlg_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_ueq(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ueq float %a, %b %1 = sext i1 %0 to i32 @@ -142,9 +148,9 @@ entry: ; FUNC-LABEL: {{^}}f32_ugt: ; R600: SETGE ; R600: SETE_DX10 -; SI: v_cmp_nle_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_ugt(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_nle_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_ugt(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ugt float %a, %b %1 = sext i1 %0 to i32 @@ -156,9 +162,9 @@ entry: ; R600: SETGT ; R600: SETE_DX10 -; SI: v_cmp_nlt_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_uge(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_nlt_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_uge(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp uge float %a, %b %1 = sext i1 %0 to i32 @@ -170,9 +176,9 @@ entry: ; R600: SETGE ; R600: SETE_DX10 -; SI: v_cmp_nge_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_ult(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_nge_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_ult(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ult float %a, %b %1 = sext i1 %0 to i32 @@ -184,9 +190,9 @@ entry: ; R600: SETGT ; R600: SETE_DX10 -; SI: v_cmp_ngt_f32_e32 vcc -; SI-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f32_ule(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_ngt_f32_e32 vcc +; GCN-NEXT: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f32_ule(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp ule float %a, %b %1 = sext i1 %0 to i32 @@ -196,8 +202,8 @@ entry: ; FUNC-LABEL: {{^}}f32_une: ; R600: SETNE_DX10 -; SI: v_cmp_neq_f32 -define void @f32_une(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_neq_f32 +define void @f32_une(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp une float %a, %b %1 = sext i1 %0 to i32 @@ -210,8 +216,8 @@ entry: ; R600: SETNE_DX10 ; R600: OR_INT ; R600: SETNE_INT -; SI: v_cmp_u_f32 -define void @f32_uno(i32 addrspace(1)* %out, float %a, float %b) { +; GCN: v_cmp_u_f32 +define void @f32_uno(i32 addrspace(1)* %out, float %a, float %b) #0 { entry: %0 = fcmp uno float %a, %b %1 = sext i1 %0 to i32 @@ -225,8 +231,8 @@ entry: ; FUNC-LABEL: {{^}}i32_eq: ; R600: SETE_INT -; SI: v_cmp_eq_i32 -define void @i32_eq(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_eq_u32 +define void @i32_eq(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp eq i32 %a, %b %1 = sext i1 %0 to i32 @@ -236,8 +242,8 @@ entry: ; FUNC-LABEL: {{^}}i32_ne: ; R600: SETNE_INT -; SI: v_cmp_ne_i32 -define void @i32_ne(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_ne_u32 +define void @i32_ne(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp ne i32 %a, %b %1 = sext i1 %0 to i32 @@ -247,8 +253,8 @@ entry: ; FUNC-LABEL: {{^}}i32_ugt: ; R600: SETGT_UINT -; SI: v_cmp_gt_u32 -define void @i32_ugt(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_gt_u32 +define void @i32_ugt(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp ugt i32 %a, %b %1 = sext i1 %0 to i32 @@ -258,8 +264,8 @@ entry: ; FUNC-LABEL: {{^}}i32_uge: ; R600: SETGE_UINT -; SI: v_cmp_ge_u32 -define void @i32_uge(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_ge_u32 +define void @i32_uge(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp uge i32 %a, %b %1 = sext i1 %0 to i32 @@ -269,8 +275,8 @@ entry: ; FUNC-LABEL: {{^}}i32_ult: ; R600: SETGT_UINT -; SI: v_cmp_lt_u32 -define void @i32_ult(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_lt_u32 +define void @i32_ult(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp ult i32 %a, %b %1 = sext i1 %0 to i32 @@ -280,8 +286,8 @@ entry: ; FUNC-LABEL: {{^}}i32_ule: ; R600: SETGE_UINT -; SI: v_cmp_le_u32 -define void @i32_ule(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_le_u32 +define void @i32_ule(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp ule i32 %a, %b %1 = sext i1 %0 to i32 @@ -291,8 +297,8 @@ entry: ; FUNC-LABEL: {{^}}i32_sgt: ; R600: SETGT_INT -; SI: v_cmp_gt_i32 -define void @i32_sgt(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_gt_i32 +define void @i32_sgt(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp sgt i32 %a, %b %1 = sext i1 %0 to i32 @@ -302,8 +308,8 @@ entry: ; FUNC-LABEL: {{^}}i32_sge: ; R600: SETGE_INT -; SI: v_cmp_ge_i32 -define void @i32_sge(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_ge_i32 +define void @i32_sge(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp sge i32 %a, %b %1 = sext i1 %0 to i32 @@ -313,8 +319,8 @@ entry: ; FUNC-LABEL: {{^}}i32_slt: ; R600: SETGT_INT -; SI: v_cmp_lt_i32 -define void @i32_slt(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_lt_i32 +define void @i32_slt(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp slt i32 %a, %b %1 = sext i1 %0 to i32 @@ -324,8 +330,8 @@ entry: ; FUNC-LABEL: {{^}}i32_sle: ; R600: SETGE_INT -; SI: v_cmp_le_i32 -define void @i32_sle(i32 addrspace(1)* %out, i32 %a, i32 %b) { +; GCN: v_cmp_le_i32 +define void @i32_sle(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: %0 = icmp sle i32 %a, %b %1 = sext i1 %0 to i32 @@ -335,14 +341,14 @@ entry: ; FIXME: This does 4 compares ; FUNC-LABEL: {{^}}v3i32_eq: -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI: s_endpgm -define void @v3i32_eq(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %ptra, <3 x i32> addrspace(1)* %ptrb) { +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN: s_endpgm +define void @v3i32_eq(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %ptra, <3 x i32> addrspace(1)* %ptrb) #0 { %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone %gep.a = getelementptr <3 x i32>, <3 x i32> addrspace(1)* %ptra, i32 %tid %gep.b = getelementptr <3 x i32>, <3 x i32> addrspace(1)* %ptrb, i32 %tid @@ -356,14 +362,14 @@ define void @v3i32_eq(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %ptr } ; FUNC-LABEL: {{^}}v3i8_eq: -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI-DAG: v_cmp_eq_i32 -; SI-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, -; SI: s_endpgm -define void @v3i8_eq(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %ptra, <3 x i8> addrspace(1)* %ptrb) { +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN-DAG: v_cmp_eq_u32 +; GCN-DAG: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, +; GCN: s_endpgm +define void @v3i8_eq(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %ptra, <3 x i8> addrspace(1)* %ptrb) #0 { %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone %gep.a = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %ptra, i32 %tid %gep.b = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %ptrb, i32 %tid @@ -378,9 +384,9 @@ define void @v3i8_eq(<3 x i8> addrspace(1)* %out, <3 x i8> addrspace(1)* %ptra, ; Make sure we don't try to emit i1 setcc ops ; FUNC-LABEL: setcc-i1 -; SI: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 1 -; SI: s_cmp_eq_i32 [[AND]], 0 -define void @setcc-i1(i32 %in) { +; GCN: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 1 +; GCN: s_cmp_eq_u32 [[AND]], 0 +define void @setcc-i1(i32 %in) #0 { %and = and i32 %in, 1 %cmp = icmp eq i32 %and, 0 br i1 %cmp, label %endif, label %if @@ -391,9 +397,9 @@ endif: } ; FUNC-LABEL: setcc-i1-and-xor -; SI-DAG: v_cmp_le_f32_e64 [[A:s\[[0-9]+:[0-9]+\]]], 0, s{{[0-9]+}} -; SI-DAG: v_cmp_ge_f32_e64 [[B:s\[[0-9]+:[0-9]+\]]], 1.0, s{{[0-9]+}} -; SI: s_and_b64 s[2:3], [[A]], [[B]] +; GCN-DAG: v_cmp_ge_f32_e64 [[A:s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} +; GCN-DAG: v_cmp_le_f32_e64 [[B:s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 1.0 +; GCN: s_and_b64 s[2:3], [[A]], [[B]] define void @setcc-i1-and-xor(i32 addrspace(1)* %out, float %cond) #0 { bb0: %tmp5 = fcmp oge float %cond, 0.000000e+00 @@ -409,3 +415,5 @@ bb1: bb2: ret void } + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/setcc64.ll b/test/CodeGen/AMDGPU/setcc64.ll index 15db03cf906e..5c6804e071df 100644 --- a/test/CodeGen/AMDGPU/setcc64.ll +++ b/test/CodeGen/AMDGPU/setcc64.ll @@ -1,5 +1,5 @@ -;RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s -;RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs| FileCheck --check-prefix=SI --check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s| FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s ; XXX: Merge this into setcc, once R600 supports 64-bit operations @@ -7,150 +7,150 @@ ;; Double comparisons ;;;==========================================================================;;; -; FUNC-LABEL: {{^}}f64_oeq: -; SI: v_cmp_eq_f64 -define void @f64_oeq(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_oeq: +; GCN: v_cmp_eq_f64 +define void @f64_oeq(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp oeq double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp oeq double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ogt: -; SI: v_cmp_gt_f64 -define void @f64_ogt(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ogt: +; GCN: v_cmp_gt_f64 +define void @f64_ogt(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ogt double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ogt double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_oge: -; SI: v_cmp_ge_f64 -define void @f64_oge(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_oge: +; GCN: v_cmp_ge_f64 +define void @f64_oge(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp oge double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp oge double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_olt: -; SI: v_cmp_lt_f64 -define void @f64_olt(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_olt: +; GCN: v_cmp_lt_f64 +define void @f64_olt(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp olt double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp olt double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ole: -; SI: v_cmp_le_f64 -define void @f64_ole(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ole: +; GCN: v_cmp_le_f64 +define void @f64_ole(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ole double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ole double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_one: -; SI: v_cmp_lg_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_one(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_one: +; GCN: v_cmp_lg_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_one(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp one double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp one double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ord: -; SI: v_cmp_o_f64 -define void @f64_ord(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ord: +; GCN: v_cmp_o_f64 +define void @f64_ord(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ord double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ord double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ueq: -; SI: v_cmp_nlg_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_ueq(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ueq: +; GCN: v_cmp_nlg_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_ueq(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ueq double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ueq double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ugt: +; GCN-LABEL: {{^}}f64_ugt: -; SI: v_cmp_nle_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_ugt(i32 addrspace(1)* %out, double %a, double %b) { +; GCN: v_cmp_nle_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_ugt(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ugt double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ugt double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_uge: -; SI: v_cmp_nlt_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_uge(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_uge: +; GCN: v_cmp_nlt_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_uge(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp uge double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp uge double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ult: -; SI: v_cmp_nge_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_ult(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ult: +; GCN: v_cmp_nge_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_ult(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ult double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ult double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_ule: -; SI: v_cmp_ngt_f64_e32 vcc -; SI: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc -define void @f64_ule(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_ule: +; GCN: v_cmp_ngt_f64_e32 vcc +; GCN: v_cndmask_b32_e64 {{v[0-9]+}}, 0, -1, vcc +define void @f64_ule(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp ule double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp ule double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_une: -; SI: v_cmp_neq_f64 -define void @f64_une(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_une: +; GCN: v_cmp_neq_f64 +define void @f64_une(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp une double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp une double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}f64_uno: -; SI: v_cmp_u_f64 -define void @f64_uno(i32 addrspace(1)* %out, double %a, double %b) { +; GCN-LABEL: {{^}}f64_uno: +; GCN: v_cmp_u_f64 +define void @f64_uno(i32 addrspace(1)* %out, double %a, double %b) #0 { entry: - %0 = fcmp uno double %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = fcmp uno double %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } @@ -158,102 +158,104 @@ entry: ;; 64-bit integer comparisons ;;;==========================================================================;;; -; FUNC-LABEL: {{^}}i64_eq: -; SI: v_cmp_eq_i64 -define void @i64_eq(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_eq: +; GCN: v_cmp_eq_u64 +define void @i64_eq(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp eq i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp eq i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_ne: -; SI: v_cmp_ne_i64 -define void @i64_ne(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_ne: +; GCN: v_cmp_ne_u64 +define void @i64_ne(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp ne i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp ne i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_ugt: -; SI: v_cmp_gt_u64 -define void @i64_ugt(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_ugt: +; GCN: v_cmp_gt_u64 +define void @i64_ugt(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp ugt i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp ugt i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_uge: -; SI: v_cmp_ge_u64 -define void @i64_uge(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_uge: +; GCN: v_cmp_ge_u64 +define void @i64_uge(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp uge i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp uge i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_ult: -; SI: v_cmp_lt_u64 -define void @i64_ult(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_ult: +; GCN: v_cmp_lt_u64 +define void @i64_ult(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp ult i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp ult i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_ule: -; SI: v_cmp_le_u64 -define void @i64_ule(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_ule: +; GCN: v_cmp_le_u64 +define void @i64_ule(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp ule i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp ule i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_sgt: -; SI: v_cmp_gt_i64 -define void @i64_sgt(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_sgt: +; GCN: v_cmp_gt_i64 +define void @i64_sgt(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp sgt i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp sgt i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_sge: -; SI: v_cmp_ge_i64 -define void @i64_sge(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_sge: +; GCN: v_cmp_ge_i64 +define void @i64_sge(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp sge i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp sge i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_slt: -; SI: v_cmp_lt_i64 -define void @i64_slt(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_slt: +; GCN: v_cmp_lt_i64 +define void @i64_slt(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp slt i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp slt i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}i64_sle: -; SI: v_cmp_le_i64 -define void @i64_sle(i32 addrspace(1)* %out, i64 %a, i64 %b) { +; GCN-LABEL: {{^}}i64_sle: +; GCN: v_cmp_le_i64 +define void @i64_sle(i32 addrspace(1)* %out, i64 %a, i64 %b) #0 { entry: - %0 = icmp sle i64 %a, %b - %1 = sext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp sle i64 %a, %b + %tmp1 = sext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/sext-in-reg-failure-r600.ll b/test/CodeGen/AMDGPU/sext-in-reg-failure-r600.ll index 08bdc3aba555..adba6bbb51d4 100644 --- a/test/CodeGen/AMDGPU/sext-in-reg-failure-r600.ll +++ b/test/CodeGen/AMDGPU/sext-in-reg-failure-r600.ll @@ -1,15 +1,13 @@ -; XFAIL: * -; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s -; XUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG %s +; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG %s ; ; EG-LABEL: {{^}}sext_in_reg_v2i1_in_v2i32_other_amount: -; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] +; EG: MEM_{{.*}} MSKOR [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG-NOT: BFE ; EG: ADD_INT ; EG: LSHL -; EG: ASHR [[RES]] +; EG: ASHR ; EG: LSHL -; EG: ASHR [[RES]] +; EG: ASHR ; EG: LSHR {{\*?}} [[ADDR]] ; Works with the align 2 removed diff --git a/test/CodeGen/AMDGPU/sext-in-reg.ll b/test/CodeGen/AMDGPU/sext-in-reg.ll index a6c72a5165d6..f9216d311471 100644 --- a/test/CodeGen/AMDGPU/sext-in-reg.ll +++ b/test/CodeGen/AMDGPU/sext-in-reg.ll @@ -1,20 +1,17 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s -declare i32 @llvm.AMDGPU.imax(i32, i32) nounwind readnone -declare i32 @llvm.r600.read.tidig.x() nounwind readnone - - ; FUNC-LABEL: {{^}}sext_in_reg_i1_i32: -; SI: s_load_dword [[ARG:s[0-9]+]], -; SI: s_bfe_i32 [[SEXTRACT:s[0-9]+]], [[ARG]], 0x10000 -; SI: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], [[SEXTRACT]] -; SI: buffer_store_dword [[EXTRACT]], +; GCN: s_load_dword [[ARG:s[0-9]+]], +; GCN: s_bfe_i32 [[SEXTRACT:s[0-9]+]], [[ARG]], 0x10000 +; GCN: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], [[SEXTRACT]] +; GCN: buffer_store_dword [[EXTRACT]], ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]] ; EG: LSHR * [[ADDR]] ; EG: BFE_INT * [[RES]], {{.*}}, 0.0, 1 -define void @sext_in_reg_i1_i32(i32 addrspace(1)* %out, i32 %in) { +define void @sext_in_reg_i1_i32(i32 addrspace(1)* %out, i32 %in) #0 { %shl = shl i32 %in, 31 %sext = ashr i32 %shl, 31 store i32 %sext, i32 addrspace(1)* %out @@ -22,16 +19,16 @@ define void @sext_in_reg_i1_i32(i32 addrspace(1)* %out, i32 %in) { } ; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i32: -; SI: s_add_i32 [[VAL:s[0-9]+]], -; SI: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]] -; SI: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] -; SI: buffer_store_dword [[VEXTRACT]], +; GCN: s_add_i32 [[VAL:s[0-9]+]], +; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] +; GCN: buffer_store_dword [[VEXTRACT]], ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]] ; EG: ADD_INT ; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal ; EG-NEXT: LSHR * [[ADDR]] -define void @sext_in_reg_i8_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +define void @sext_in_reg_i8_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { %c = add i32 %a, %b ; add to prevent folding into extload %shl = shl i32 %c, 24 %ashr = ashr i32 %shl, 24 @@ -40,16 +37,16 @@ define void @sext_in_reg_i8_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounw } ; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i32: -; SI: s_add_i32 [[VAL:s[0-9]+]], -; SI: s_sext_i32_i16 [[EXTRACT:s[0-9]+]], [[VAL]] -; SI: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] -; SI: buffer_store_dword [[VEXTRACT]], +; GCN: s_add_i32 [[VAL:s[0-9]+]], +; GCN: s_sext_i32_i16 [[EXTRACT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] +; GCN: buffer_store_dword [[VEXTRACT]], ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]] ; EG: ADD_INT ; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal ; EG-NEXT: LSHR * [[ADDR]] -define void @sext_in_reg_i16_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +define void @sext_in_reg_i16_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { %c = add i32 %a, %b ; add to prevent folding into extload %shl = shl i32 %c, 16 %ashr = ashr i32 %shl, 16 @@ -58,16 +55,16 @@ define void @sext_in_reg_i16_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) noun } ; FUNC-LABEL: {{^}}sext_in_reg_i8_to_v1i32: -; SI: s_add_i32 [[VAL:s[0-9]+]], -; SI: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]] -; SI: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] -; SI: buffer_store_dword [[VEXTRACT]], +; GCN: s_add_i32 [[VAL:s[0-9]+]], +; GCN: s_sext_i32_i8 [[EXTRACT:s[0-9]+]], [[VAL]] +; GCN: v_mov_b32_e32 [[VEXTRACT:v[0-9]+]], [[EXTRACT]] +; GCN: buffer_store_dword [[VEXTRACT]], ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]] ; EG: ADD_INT ; EG-NEXT: BFE_INT [[RES]], {{.*}}, 0.0, literal ; EG-NEXT: LSHR * [[ADDR]] -define void @sext_in_reg_i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i32> %a, <1 x i32> %b) nounwind { +define void @sext_in_reg_i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i32> %a, <1 x i32> %b) #0 { %c = add <1 x i32> %a, %b ; add to prevent folding into extload %shl = shl <1 x i32> %c, <i32 24> %ashr = ashr <1 x i32> %shl, <i32 24> @@ -76,12 +73,12 @@ define void @sext_in_reg_i8_to_v1i32(<1 x i32> addrspace(1)* %out, <1 x i32> %a, } ; FUNC-LABEL: {{^}}sext_in_reg_i1_to_i64: -; SI: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] -; SI-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x10000 -; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] -; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] -; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} -define void @sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind { +; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] +; GCN-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x10000 +; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] +; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +define void @sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) #0 { %c = shl i64 %a, %b %shl = shl i64 %c, 63 %ashr = ashr i64 %shl, 63 @@ -90,12 +87,12 @@ define void @sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounw } ; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i64: -; SI: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] -; SI-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x80000 -; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] -; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] -; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} -define void @sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind { +; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] +; GCN-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x80000 +; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] +; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +define void @sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) #0 { %c = shl i64 %a, %b %shl = shl i64 %c, 56 %ashr = ashr i64 %shl, 56 @@ -104,13 +101,13 @@ define void @sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounw } ; FUNC-LABEL: {{^}}sext_in_reg_i16_to_i64: -; SI: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] -; SI-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x100000 -; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] -; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] -; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] +; GCN-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x100000 +; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] +; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} -define void @sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind { +define void @sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) #0 { %c = shl i64 %a, %b %shl = shl i64 %c, 48 %ashr = ashr i64 %shl, 48 @@ -119,12 +116,12 @@ define void @sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) noun } ; FUNC-LABEL: {{^}}sext_in_reg_i32_to_i64: -; SI: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] -; SI-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x200000 -; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] -; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] -; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} -define void @sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind { +; GCN: s_lshl_b64 [[VAL:s\[[0-9]+:[0-9]+\]]] +; GCN-DAG: s_bfe_i64 s{{\[}}[[SLO:[0-9]+]]:[[SHI:[0-9]+]]{{\]}}, [[VAL]], 0x200000 +; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]] +; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]] +; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +define void @sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) #0 { %c = shl i64 %a, %b %shl = shl i64 %c, 32 %ashr = ashr i64 %shl, 32 @@ -134,12 +131,12 @@ define void @sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) noun ; This is broken on Evergreen for some reason related to the <1 x i64> kernel arguments. ; XFUNC-LABEL: {{^}}sext_in_reg_i8_to_v1i64: -; XSI: s_bfe_i32 [[EXTRACT:s[0-9]+]], {{s[0-9]+}}, 524288 -; XSI: s_ashr_i32 {{v[0-9]+}}, [[EXTRACT]], 31 -; XSI: buffer_store_dword +; XGCN: s_bfe_i32 [[EXTRACT:s[0-9]+]], {{s[0-9]+}}, 524288 +; XGCN: s_ashr_i32 {{v[0-9]+}}, [[EXTRACT]], 31 +; XGCN: buffer_store_dword ; XEG: BFE_INT ; XEG: ASHR -; define void @sext_in_reg_i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i64> %a, <1 x i64> %b) nounwind { +; define void @sext_in_reg_i8_to_v1i64(<1 x i64> addrspace(1)* %out, <1 x i64> %a, <1 x i64> %b) #0 { ; %c = add <1 x i64> %a, %b ; %shl = shl <1 x i64> %c, <i64 56> ; %ashr = ashr <1 x i64> %shl, <i64 56> @@ -150,10 +147,16 @@ define void @sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) noun ; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} -; SI: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1 -; SI: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} + +; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1 +; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} -define void @v_sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind { +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @v_sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -171,10 +174,16 @@ define void @v_sext_in_reg_i1_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* % ; FUNC-LABEL: {{^}}v_sext_in_reg_i8_to_i64: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} -; SI: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 8 -; SI: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} + +; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 8 +; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} -define void @v_sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind { +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @v_sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -192,10 +201,16 @@ define void @v_sext_in_reg_i8_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* % ; FUNC-LABEL: {{^}}v_sext_in_reg_i16_to_i64: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} -; SI: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 16 -; SI: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} + +; GCN: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 16 +; GCN: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] + ; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} -define void @v_sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind { +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @v_sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -213,9 +228,13 @@ define void @v_sext_in_reg_i16_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* ; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, -; SI: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]] -; SI: buffer_store_dwordx2 v{{\[}}[[LO]]:[[SHR]]{{\]}} -define void @v_sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) nounwind { + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, + +; GCN: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]] +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO]]:[[SHR]]{{\]}} +define void @v_sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -231,9 +250,9 @@ define void @v_sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* } ; FUNC-LABEL: {{^}}sext_in_reg_i1_in_i32_other_amount: -; SI-NOT: s_lshl -; SI-NOT: s_ashr -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 +; GCN-NOT: s_lshl +; GCN-NOT: s_ashr +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+\.[XYZW]]], [[ADDR:T[0-9]+.[XYZW]]] ; EG-NOT: BFE @@ -241,7 +260,7 @@ define void @v_sext_in_reg_i32_to_i64(i64 addrspace(1)* %out, i64 addrspace(1)* ; EG: LSHL ; EG: ASHR [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_i1_in_i32_other_amount(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +define void @sext_in_reg_i1_in_i32_other_amount(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { %c = add i32 %a, %b %x = shl i32 %c, 6 %y = ashr i32 %x, 7 @@ -250,11 +269,11 @@ define void @sext_in_reg_i1_in_i32_other_amount(i32 addrspace(1)* %out, i32 %a, } ; FUNC-LABEL: {{^}}sext_in_reg_v2i1_in_v2i32_other_amount: -; SI-NOT: s_lshl -; SI-NOT: s_ashr -; SI-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 -; SI-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 -; SI: s_endpgm +; GCN-NOT: s_lshl +; GCN-NOT: s_ashr +; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 +; GCN-DAG: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x190001 +; GCN: s_endpgm ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG-NOT: BFE @@ -264,7 +283,7 @@ define void @sext_in_reg_i1_in_i32_other_amount(i32 addrspace(1)* %out, i32 %a, ; EG: LSHL ; EG: ASHR [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v2i1_in_v2i32_other_amount(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) nounwind { +define void @sext_in_reg_v2i1_in_v2i32_other_amount(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) #0 { %c = add <2 x i32> %a, %b %x = shl <2 x i32> %c, <i32 6, i32 6> %y = ashr <2 x i32> %x, <i32 7, i32 7> @@ -274,15 +293,15 @@ define void @sext_in_reg_v2i1_in_v2i32_other_amount(<2 x i32> addrspace(1)* %out ; FUNC-LABEL: {{^}}sext_in_reg_v2i1_to_v2i32: -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: buffer_store_dwordx2 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: buffer_store_dwordx2 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG: BFE_INT [[RES]] ; EG: BFE_INT [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v2i1_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) nounwind { +define void @sext_in_reg_v2i1_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) #0 { %c = add <2 x i32> %a, %b ; add to prevent folding into extload %shl = shl <2 x i32> %c, <i32 31, i32 31> %ashr = ashr <2 x i32> %shl, <i32 31, i32 31> @@ -291,11 +310,11 @@ define void @sext_in_reg_v2i1_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> % } ; FUNC-LABEL: {{^}}sext_in_reg_v4i1_to_v4i32: -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 -; SI: buffer_store_dwordx4 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: s_bfe_i32 {{s[0-9]+}}, {{s[0-9]+}}, 0x10000 +; GCN: buffer_store_dwordx4 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG: BFE_INT [[RES]] @@ -303,7 +322,7 @@ define void @sext_in_reg_v2i1_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> % ; EG: BFE_INT [[RES]] ; EG: BFE_INT [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v4i1_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b) nounwind { +define void @sext_in_reg_v4i1_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b) #0 { %c = add <4 x i32> %a, %b ; add to prevent folding into extload %shl = shl <4 x i32> %c, <i32 31, i32 31, i32 31, i32 31> %ashr = ashr <4 x i32> %shl, <i32 31, i32 31, i32 31, i32 31> @@ -312,15 +331,15 @@ define void @sext_in_reg_v4i1_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> % } ; FUNC-LABEL: {{^}}sext_in_reg_v2i8_to_v2i32: -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: buffer_store_dwordx2 +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: buffer_store_dwordx2 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG: BFE_INT [[RES]] ; EG: BFE_INT [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) nounwind { +define void @sext_in_reg_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) #0 { %c = add <2 x i32> %a, %b ; add to prevent folding into extload %shl = shl <2 x i32> %c, <i32 24, i32 24> %ashr = ashr <2 x i32> %shl, <i32 24, i32 24> @@ -329,11 +348,11 @@ define void @sext_in_reg_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> % } ; FUNC-LABEL: {{^}}sext_in_reg_v4i8_to_v4i32: -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} -; SI: buffer_store_dwordx4 +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: s_sext_i32_i8 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: buffer_store_dwordx4 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW][XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG: BFE_INT [[RES]] @@ -341,7 +360,7 @@ define void @sext_in_reg_v2i8_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> % ; EG: BFE_INT [[RES]] ; EG: BFE_INT [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b) nounwind { +define void @sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %a, <4 x i32> %b) #0 { %c = add <4 x i32> %a, %b ; add to prevent folding into extload %shl = shl <4 x i32> %c, <i32 24, i32 24, i32 24, i32 24> %ashr = ashr <4 x i32> %shl, <i32 24, i32 24, i32 24, i32 24> @@ -350,15 +369,15 @@ define void @sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> % } ; FUNC-LABEL: {{^}}sext_in_reg_v2i16_to_v2i32: -; SI: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}} -; SI: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}} -; SI: buffer_store_dwordx2 +; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: s_sext_i32_i16 {{s[0-9]+}}, {{s[0-9]+}} +; GCN: buffer_store_dwordx2 ; EG: MEM_{{.*}} STORE_{{.*}} [[RES:T[0-9]+]]{{\.[XYZW][XYZW]}}, [[ADDR:T[0-9]+.[XYZW]]] ; EG: BFE_INT [[RES]] ; EG: BFE_INT [[RES]] ; EG: LSHR {{\*?}} [[ADDR]] -define void @sext_in_reg_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) nounwind { +define void @sext_in_reg_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a, <2 x i32> %b) #0 { %c = add <2 x i32> %a, %b ; add to prevent folding into extload %shl = shl <2 x i32> %c, <i32 16, i32 16> %ashr = ashr <2 x i32> %shl, <i32 16, i32 16> @@ -367,7 +386,7 @@ define void @sext_in_reg_v2i16_to_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> } ; FUNC-LABEL: {{^}}testcase: -define void @testcase(i8 addrspace(1)* %out, i8 %a) nounwind { +define void @testcase(i8 addrspace(1)* %out, i8 %a) #0 { %and_a_1 = and i8 %a, 1 %cmp_eq = icmp eq i8 %and_a_1, 0 %cmp_slt = icmp slt i8 %a, 0 @@ -379,7 +398,7 @@ define void @testcase(i8 addrspace(1)* %out, i8 %a) nounwind { } ; FUNC-LABEL: {{^}}testcase_3: -define void @testcase_3(i8 addrspace(1)* %out, i8 %a) nounwind { +define void @testcase_3(i8 addrspace(1)* %out, i8 %a) #0 { %and_a_1 = and i8 %a, 1 %cmp_eq = icmp eq i8 %and_a_1, 0 %cmp_slt = icmp slt i8 %a, 0 @@ -391,11 +410,11 @@ define void @testcase_3(i8 addrspace(1)* %out, i8 %a) nounwind { } ; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i8_to_v4i32: -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 -define void @vgpr_sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %a, <4 x i32> addrspace(1)* %b) nounwind { +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 8 +define void @vgpr_sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %a, <4 x i32> addrspace(1)* %b) #0 { %loada = load <4 x i32>, <4 x i32> addrspace(1)* %a, align 16 %loadb = load <4 x i32>, <4 x i32> addrspace(1)* %b, align 16 %c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload @@ -406,9 +425,9 @@ define void @vgpr_sext_in_reg_v4i8_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i } ; FUNC-LABEL: {{^}}vgpr_sext_in_reg_v4i16_to_v4i32: -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16 -; SI: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16 -define void @vgpr_sext_in_reg_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %a, <4 x i32> addrspace(1)* %b) nounwind { +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16 +; GCN: v_bfe_i32 [[EXTRACT:v[0-9]+]], {{v[0-9]+}}, 0, 16 +define void @vgpr_sext_in_reg_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %a, <4 x i32> addrspace(1)* %b) #0 { %loada = load <4 x i32>, <4 x i32> addrspace(1)* %a, align 16 %loadb = load <4 x i32>, <4 x i32> addrspace(1)* %b, align 16 %c = add <4 x i32> %loada, %loadb ; add to prevent folding into extload @@ -419,11 +438,11 @@ define void @vgpr_sext_in_reg_v4i16_to_v4i32(<4 x i32> addrspace(1)* %out, <4 x } ; FUNC-LABEL: {{^}}sext_in_reg_to_illegal_type: -; SI: buffer_load_sbyte -; SI: v_max_i32 -; SI-NOT: bfe -; SI: buffer_store_short -define void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %src) nounwind { +; GCN: buffer_load_sbyte +; GCN: v_max_i32 +; GCN-NOT: bfe +; GCN: buffer_store_short +define void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocapture %out, i8 addrspace(1)* nocapture %src) #0 { %tmp5 = load i8, i8 addrspace(1)* %src, align 1 %tmp2 = sext i8 %tmp5 to i32 %tmp2.5 = icmp sgt i32 %tmp2, 0 @@ -437,9 +456,9 @@ define void @sext_in_reg_to_illegal_type(i16 addrspace(1)* nocapture %out, i8 ad declare i32 @llvm.AMDGPU.bfe.i32(i32, i32, i32) nounwind readnone ; FUNC-LABEL: {{^}}bfe_0_width: -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @bfe_0_width(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwind { +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @bfe_0_width(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { %load = load i32, i32 addrspace(1)* %ptr, align 4 %bfe = call i32 @llvm.AMDGPU.bfe.i32(i32 %load, i32 8, i32 0) nounwind readnone store i32 %bfe, i32 addrspace(1)* %out, align 4 @@ -447,10 +466,10 @@ define void @bfe_0_width(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwin } ; FUNC-LABEL: {{^}}bfe_8_bfe_8: -; SI: v_bfe_i32 -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @bfe_8_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwind { +; GCN: v_bfe_i32 +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @bfe_8_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { %load = load i32, i32 addrspace(1)* %ptr, align 4 %bfe0 = call i32 @llvm.AMDGPU.bfe.i32(i32 %load, i32 0, i32 8) nounwind readnone %bfe1 = call i32 @llvm.AMDGPU.bfe.i32(i32 %bfe0, i32 0, i32 8) nounwind readnone @@ -459,9 +478,9 @@ define void @bfe_8_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwin } ; FUNC-LABEL: {{^}}bfe_8_bfe_16: -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 -; SI: s_endpgm -define void @bfe_8_bfe_16(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwind { +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 +; GCN: s_endpgm +define void @bfe_8_bfe_16(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { %load = load i32, i32 addrspace(1)* %ptr, align 4 %bfe0 = call i32 @llvm.AMDGPU.bfe.i32(i32 %load, i32 0, i32 8) nounwind readnone %bfe1 = call i32 @llvm.AMDGPU.bfe.i32(i32 %bfe0, i32 0, i32 16) nounwind readnone @@ -471,10 +490,10 @@ define void @bfe_8_bfe_16(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwi ; This really should be folded into 1 ; FUNC-LABEL: {{^}}bfe_16_bfe_8: -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @bfe_16_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwind { +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 8 +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @bfe_16_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) #0 { %load = load i32, i32 addrspace(1)* %ptr, align 4 %bfe0 = call i32 @llvm.AMDGPU.bfe.i32(i32 %load, i32 0, i32 16) nounwind readnone %bfe1 = call i32 @llvm.AMDGPU.bfe.i32(i32 %bfe0, i32 0, i32 8) nounwind readnone @@ -484,10 +503,10 @@ define void @bfe_16_bfe_8(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) nounwi ; Make sure there isn't a redundant BFE ; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i32_bfe: -; SI: s_sext_i32_i8 s{{[0-9]+}}, s{{[0-9]+}} -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @sext_in_reg_i8_to_i32_bfe(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +; GCN: s_sext_i32_i8 s{{[0-9]+}}, s{{[0-9]+}} +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @sext_in_reg_i8_to_i32_bfe(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { %c = add i32 %a, %b ; add to prevent folding into extload %bfe = call i32 @llvm.AMDGPU.bfe.i32(i32 %c, i32 0, i32 8) nounwind readnone %shl = shl i32 %bfe, 24 @@ -497,7 +516,7 @@ define void @sext_in_reg_i8_to_i32_bfe(i32 addrspace(1)* %out, i32 %a, i32 %b) n } ; FUNC-LABEL: {{^}}sext_in_reg_i8_to_i32_bfe_wrong: -define void @sext_in_reg_i8_to_i32_bfe_wrong(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +define void @sext_in_reg_i8_to_i32_bfe_wrong(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { %c = add i32 %a, %b ; add to prevent folding into extload %bfe = call i32 @llvm.AMDGPU.bfe.i32(i32 %c, i32 8, i32 0) nounwind readnone %shl = shl i32 %bfe, 24 @@ -507,10 +526,10 @@ define void @sext_in_reg_i8_to_i32_bfe_wrong(i32 addrspace(1)* %out, i32 %a, i32 } ; FUNC-LABEL: {{^}}sextload_i8_to_i32_bfe: -; SI: buffer_load_sbyte -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @sextload_i8_to_i32_bfe(i32 addrspace(1)* %out, i8 addrspace(1)* %ptr) nounwind { +; GCN: buffer_load_sbyte +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @sextload_i8_to_i32_bfe(i32 addrspace(1)* %out, i8 addrspace(1)* %ptr) #0 { %load = load i8, i8 addrspace(1)* %ptr, align 1 %sext = sext i8 %load to i32 %bfe = call i32 @llvm.AMDGPU.bfe.i32(i32 %sext, i32 0, i32 8) nounwind readnone @@ -520,11 +539,11 @@ define void @sextload_i8_to_i32_bfe(i32 addrspace(1)* %out, i8 addrspace(1)* %pt ret void } -; SI: .text +; GCN: .text ; FUNC-LABEL: {{^}}sextload_i8_to_i32_bfe_0:{{.*$}} -; SI-NOT: {{[^@]}}bfe -; SI: s_endpgm -define void @sextload_i8_to_i32_bfe_0(i32 addrspace(1)* %out, i8 addrspace(1)* %ptr) nounwind { +; GCN-NOT: {{[^@]}}bfe +; GCN: s_endpgm +define void @sextload_i8_to_i32_bfe_0(i32 addrspace(1)* %out, i8 addrspace(1)* %ptr) #0 { %load = load i8, i8 addrspace(1)* %ptr, align 1 %sext = sext i8 %load to i32 %bfe = call i32 @llvm.AMDGPU.bfe.i32(i32 %sext, i32 8, i32 0) nounwind readnone @@ -535,11 +554,11 @@ define void @sextload_i8_to_i32_bfe_0(i32 addrspace(1)* %out, i8 addrspace(1)* % } ; FUNC-LABEL: {{^}}sext_in_reg_i1_bfe_offset_0: -; SI-NOT: shr -; SI-NOT: shl -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 1 -; SI: s_endpgm -define void @sext_in_reg_i1_bfe_offset_0(i32 addrspace(1)* %out, i32 addrspace(1)* %in) nounwind { +; GCN-NOT: shr +; GCN-NOT: shl +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 1 +; GCN: s_endpgm +define void @sext_in_reg_i1_bfe_offset_0(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { %x = load i32, i32 addrspace(1)* %in, align 4 %shl = shl i32 %x, 31 %shr = ashr i32 %shl, 31 @@ -549,12 +568,12 @@ define void @sext_in_reg_i1_bfe_offset_0(i32 addrspace(1)* %out, i32 addrspace(1 } ; FUNC-LABEL: {{^}}sext_in_reg_i1_bfe_offset_1: -; SI: buffer_load_dword -; SI-NOT: shl -; SI-NOT: shr -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 1, 1 -; SI: s_endpgm -define void @sext_in_reg_i1_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) nounwind { +; GCN: buffer_load_dword +; GCN-NOT: shl +; GCN-NOT: shr +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 1, 1 +; GCN: s_endpgm +define void @sext_in_reg_i1_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { %x = load i32, i32 addrspace(1)* %in, align 4 %shl = shl i32 %x, 30 %shr = ashr i32 %shl, 30 @@ -564,13 +583,13 @@ define void @sext_in_reg_i1_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1 } ; FUNC-LABEL: {{^}}sext_in_reg_i2_bfe_offset_1: -; SI: buffer_load_dword -; SI-NOT: v_lshl -; SI-NOT: v_ashr -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 2 -; SI: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 1, 2 -; SI: s_endpgm -define void @sext_in_reg_i2_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) nounwind { +; GCN: buffer_load_dword +; GCN-NOT: v_lshl +; GCN-NOT: v_ashr +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 2 +; GCN: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 1, 2 +; GCN: s_endpgm +define void @sext_in_reg_i2_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 { %x = load i32, i32 addrspace(1)* %in, align 4 %shl = shl i32 %x, 30 %shr = ashr i32 %shl, 30 @@ -584,12 +603,17 @@ define void @sext_in_reg_i2_bfe_offset_1(i32 addrspace(1)* %out, i32 addrspace(1 ; FUNC-LABEL: {{^}}v_sext_in_reg_i1_to_i64_move_use: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} -; SI-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1 -; SI-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] -; SI-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]] -; SI-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[HI]] + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}} + +; GCN-DAG: v_bfe_i32 v[[LO:[0-9]+]], v[[VAL_LO]], 0, 1 +; GCN-DAG: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] +; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]] +; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[HI]] ; SI: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} -define void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) nounwind { +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} +define void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -609,11 +633,17 @@ define void @v_sext_in_reg_i1_to_i64_move_use(i64 addrspace(1)* %out, i64 addrsp ; FUNC-LABEL: {{^}}v_sext_in_reg_i32_to_i64_move_use: ; SI: buffer_load_dwordx2 ; SI: v_lshl_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, -; SI-DAG: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]] -; SI-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]] -; SI-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[SHR]] + +; VI: flat_load_dwordx2 +; VI: v_lshlrev_b64 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, + +; GCN-DAG: v_ashrrev_i32_e32 v[[SHR:[0-9]+]], 31, v[[LO]] +; GCN-DAG: v_and_b32_e32 v[[RESULT_LO:[0-9]+]], s{{[0-9]+}}, v[[LO]] +; GCN-DAG: v_and_b32_e32 v[[RESULT_HI:[0-9]+]], s{{[0-9]+}}, v[[SHR]] + ; SI: buffer_store_dwordx2 v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} -define void @v_sext_in_reg_i32_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) nounwind { +; VI: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[RESULT_LO]]:[[RESULT_HI]]{{\]}} +define void @v_sext_in_reg_i32_to_i64_move_use(i64 addrspace(1)* %out, i64 addrspace(1)* %aptr, i64 addrspace(1)* %bptr, i64 %s.val) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %a.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid %b.gep = getelementptr i64, i64 addrspace(1)* %aptr, i32 %tid @@ -628,3 +658,8 @@ define void @v_sext_in_reg_i32_to_i64_move_use(i64 addrspace(1)* %out, i64 addrs store i64 %and, i64 addrspace(1)* %out.gep, align 8 ret void } + +declare i32 @llvm.r600.read.tidig.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/test/CodeGen/AMDGPU/sgpr-control-flow.ll index f1b8e8eec85d..d5d2f6b717f9 100644 --- a/test/CodeGen/AMDGPU/sgpr-control-flow.ll +++ b/test/CodeGen/AMDGPU/sgpr-control-flow.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s ; ; ; Most SALU instructions ignore control flow, so we need to make sure @@ -9,7 +9,9 @@ ; about instructions in different blocks overwriting each other. ; SI-LABEL: {{^}}sgpr_if_else_salu_br: ; SI: s_add -; SI: s_add +; SI: s_branch + +; SI: s_sub define void @sgpr_if_else_salu_br(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) { entry: @@ -17,6 +19,45 @@ entry: br i1 %0, label %if, label %else if: + %1 = sub i32 %b, %c + br label %endif + +else: + %2 = add i32 %d, %e + br label %endif + +endif: + %3 = phi i32 [%1, %if], [%2, %else] + %4 = add i32 %3, %a + store i32 %4, i32 addrspace(1)* %out + ret void +} + +; SI-LABEL: {{^}}sgpr_if_else_salu_br_opt: +; SI: s_cmp_lg_u32 +; SI: s_cbranch_scc0 [[IF:BB[0-9]+_[0-9]+]] + +; SI: ; BB#1: ; %else +; SI: s_load_dword [[LOAD0:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xe +; SI: s_load_dword [[LOAD1:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xf +; SI-NOT: add +; SI: s_branch [[ENDIF:BB[0-9]+_[0-9]+]] + +; SI: [[IF]]: ; %if +; SI: s_load_dword [[LOAD0]], s{{\[[0-9]+:[0-9]+\]}}, 0xc +; SI: s_load_dword [[LOAD1]], s{{\[[0-9]+:[0-9]+\]}}, 0xd +; SI-NOT: add + +; SI: [[ENDIF]]: ; %endif +; SI: s_add_i32 s{{[0-9]+}}, [[LOAD0]], [[LOAD1]] +; SI: buffer_store_dword +; SI-NEXT: s_endpgm +define void @sgpr_if_else_salu_br_opt(i32 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) { +entry: + %0 = icmp eq i32 %a, 0 + br i1 %0, label %if, label %else + +if: %1 = add i32 %b, %c br label %endif @@ -67,12 +108,12 @@ endif: ; SI: v_cmp_gt_i32_e32 [[CMP_IF:vcc]], 0, [[AVAL]] ; SI: v_cndmask_b32_e64 [[V_CMP:v[0-9]+]], 0, -1, [[CMP_IF]] -; SI: BB2_2: +; SI: BB{{[0-9]+}}_2: ; SI: buffer_load_dword [[AVAL:v[0-9]+]] -; SI: v_cmp_eq_i32_e32 [[CMP_ELSE:vcc]], 0, [[AVAL]] +; SI: v_cmp_eq_u32_e32 [[CMP_ELSE:vcc]], 0, [[AVAL]] ; SI: v_cndmask_b32_e64 [[V_CMP]], 0, -1, [[CMP_ELSE]] -; SI: v_cmp_ne_i32_e32 [[CMP_CMP:vcc]], 0, [[V_CMP]] +; SI: v_cmp_ne_u32_e32 [[CMP_CMP:vcc]], 0, [[V_CMP]] ; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1, [[CMP_CMP]] ; SI: buffer_store_dword [[RESULT]] define void @sgpr_if_else_valu_cmp_phi_br(i32 addrspace(1)* %out, i32 addrspace(1)* %a, i32 addrspace(1)* %b) { diff --git a/test/CodeGen/AMDGPU/sgpr-copy.ll b/test/CodeGen/AMDGPU/sgpr-copy.ll index da270c533ece..31ce4a681992 100644 --- a/test/CodeGen/AMDGPU/sgpr-copy.ll +++ b/test/CodeGen/AMDGPU/sgpr-copy.ll @@ -1,5 +1,5 @@ -; RUN: llc < %s -march=amdgcn -mcpu=SI -verify-machineinstrs | FileCheck %s -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s ; This test checks that no VGPR to SGPR copies are created by the register ; allocator. @@ -223,8 +223,15 @@ declare i32 @llvm.SI.packf16(float, float) #1 ; an assertion failure. ; CHECK-LABEL: {{^}}sample_v3: -; CHECK: image_sample -; CHECK: image_sample +; CHECK: v_mov_b32_e32 v[[SAMPLE_LO:[0-9]+]], 11 +; CHECK: v_mov_b32_e32 v[[SAMPLE_HI:[0-9]+]], 13 +; CHECK: s_branch + +; CHECK-DAG: v_mov_b32_e32 v[[SAMPLE_LO:[0-9]+]], 5 +; CHECK-DAG: v_mov_b32_e32 v[[SAMPLE_HI:[0-9]+]], 7 + +; CHECK: BB{{[0-9]+_[0-9]+}}: +; CHECK: image_sample v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[SAMPLE_LO]]:[[SAMPLE_HI]]{{\]}} ; CHECK: exp ; CHECK: s_endpgm define amdgpu_ps void @sample_v3([17 x <16 x i8>] addrspace(2)* byval %arg, [32 x <16 x i8>] addrspace(2)* byval %arg1, [16 x <8 x i32>] addrspace(2)* byval %arg2, float inreg %arg3, i32 inreg %arg4, <2 x i32> %arg5, <2 x i32> %arg6, <2 x i32> %arg7, <3 x i32> %arg8, <2 x i32> %arg9, <2 x i32> %arg10, <2 x i32> %arg11, float %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, float %arg19, float %arg20) #0 { @@ -241,14 +248,14 @@ entry: br i1 %tmp27, label %if, label %else if: ; preds = %entry - %val.if = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> zeroinitializer, <8 x i32> %tmp24, <4 x i32> %tmp26.bc, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %val.if = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> <i32 11, i32 13>, <8 x i32> %tmp24, <4 x i32> %tmp26.bc, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) %val.if.0 = extractelement <4 x float> %val.if, i32 0 %val.if.1 = extractelement <4 x float> %val.if, i32 1 %val.if.2 = extractelement <4 x float> %val.if, i32 2 br label %endif else: ; preds = %entry - %val.else = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> <i32 1, i32 0>, <8 x i32> %tmp24, <4 x i32> %tmp26.bc, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %val.else = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> <i32 5, i32 7>, <8 x i32> %tmp24, <4 x i32> %tmp26.bc, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) %val.else.0 = extractelement <4 x float> %val.else, i32 0 %val.else.1 = extractelement <4 x float> %val.else, i32 1 %val.else.2 = extractelement <4 x float> %val.else, i32 2 @@ -317,9 +324,15 @@ ENDIF69: ; preds = %LOOP68 ; This test checks that image_sample resource descriptors aren't loaded into ; vgprs. The verifier will fail if this happens. -; CHECK-LABEL:{{^}}sample_rsrc: -; CHECK: image_sample -; CHECK: image_sample +; CHECK-LABEL:{{^}}sample_rsrc + +; CHECK: s_cmp_eq_u32 +; CHECK: s_cbranch_scc0 [[END:BB[0-9]+_[0-9]+]] + +; CHECK: v_add_i32_e32 v[[ADD:[0-9]+]], vcc, 1, v{{[0-9]+}} + +; [[END]]: +; CHECK: image_sample v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+}}:[[ADD]]{{\]}} ; CHECK: s_endpgm define amdgpu_ps void @sample_rsrc([6 x <16 x i8>] addrspace(2)* byval %arg, [17 x <16 x i8>] addrspace(2)* byval %arg1, [16 x <4 x i32>] addrspace(2)* byval %arg2, [32 x <8 x i32>] addrspace(2)* byval %arg3, float inreg %arg4, i32 inreg %arg5, <2 x i32> %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <3 x i32> %arg9, <2 x i32> %arg10, <2 x i32> %arg11, <2 x i32> %arg12, float %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, i32 %arg19, float %arg20, float %arg21) #0 { bb: @@ -399,5 +412,6 @@ attributes #1 = { nounwind readnone } attributes #2 = { nounwind readonly } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !3} !2 = !{!1, !1, i64 0} +!3 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll b/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll index a6555a197388..48bbc32abcbb 100644 --- a/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll +++ b/test/CodeGen/AMDGPU/shift-and-i128-ubfe.ll @@ -97,7 +97,6 @@ define void @v_uextract_bit_127_i128(i128 addrspace(1)* %out, i128 addrspace(1)* ; GCN-DAG: v_bfe_u32 v[[ELT2PART:[0-9]+]], v[[VAL3]], 2, 2{{$}} ; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} ; GCN-DAG: v_or_b32_e32 v[[OR0:[0-9]+]], v[[SHLLO]], v[[ELT1PART]] -; GCN-DAG: v_or_b32_e32 v[[OR1:[0-9]+]], 0, v[[SHLHI]]{{$}} ; GCN-DAG: buffer_store_dwordx4 v{{\[}}[[OR0]]:[[ZERO]]{{\]}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll b/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll index c5dbfd9589a6..b85714ea54c1 100644 --- a/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll +++ b/test/CodeGen/AMDGPU/shift-and-i64-ubfe.ll @@ -341,11 +341,12 @@ define void @v_uextract_bit_27_29_multi_use_shift_i64(i64 addrspace(1)* %out, i6 ; GCN-LABEL: {{^}}v_uextract_bit_34_37_multi_use_shift_i64: ; GCN: buffer_load_dword [[VAL:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4{{$}} +; GCN: v_mov_b32_e32 v[[ZERO_SHR:[0-9]+]], 0{{$}} +; GCN: v_mov_b32_e32 v[[ZERO_BFE:[0-9]+]], v[[ZERO_SHR]] ; GCN-DAG: v_lshrrev_b32_e32 v[[SHR:[0-9]+]], 2, [[VAL]] ; GCN-DAG: v_bfe_u32 v[[BFE:[0-9]+]], [[VAL]], 2, 3 -; GCN-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} -; GCN-DAG: buffer_store_dwordx2 v{{\[}}[[SHR]]:[[ZERO]]{{\]}} -; GCN: buffer_store_dwordx2 v{{\[}}[[BFE]]:[[ZERO]]{{\]}} +; GCN-DAG: buffer_store_dwordx2 v{{\[}}[[SHR]]:[[ZERO_SHR]]{{\]}} +; GCN: buffer_store_dwordx2 v{{\[}}[[BFE]]:[[ZERO_BFE]]{{\]}} define void @v_uextract_bit_34_37_multi_use_shift_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) #1 { %id.x = tail call i32 @llvm.amdgcn.workitem.id.x() %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %id.x diff --git a/test/CodeGen/AMDGPU/shl.ll b/test/CodeGen/AMDGPU/shl.ll index 5a2b03bff990..486465cb7172 100644 --- a/test/CodeGen/AMDGPU/shl.ll +++ b/test/CodeGen/AMDGPU/shl.ll @@ -19,8 +19,8 @@ declare i32 @llvm.r600.read.tidig.x() #0 define void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in) { %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 - %a = load <2 x i32>, <2 x i32> addrspace(1) * %in - %b = load <2 x i32>, <2 x i32> addrspace(1) * %b_ptr + %a = load <2 x i32>, <2 x i32> addrspace(1)* %in + %b = load <2 x i32>, <2 x i32> addrspace(1)* %b_ptr %result = shl <2 x i32> %a, %b store <2 x i32> %result, <2 x i32> addrspace(1)* %out ret void @@ -46,13 +46,105 @@ define void @shl_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %in define void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { %b_ptr = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %in, i32 1 - %a = load <4 x i32>, <4 x i32> addrspace(1) * %in - %b = load <4 x i32>, <4 x i32> addrspace(1) * %b_ptr + %a = load <4 x i32>, <4 x i32> addrspace(1)* %in + %b = load <4 x i32>, <4 x i32> addrspace(1)* %b_ptr %result = shl <4 x i32> %a, %b store <4 x i32> %result, <4 x i32> addrspace(1)* %out ret void } +; GCN-LABEL: {{^}}shl_i16: +; SI: v_lshlrev_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} + +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @shl_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) { + %b_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1 + %a = load i16, i16 addrspace(1)* %in + %b = load i16, i16 addrspace(1)* %b_ptr + %result = shl i16 %a, %b + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_i16_v_s: +; VI: v_lshlrev_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} + +; VI: v_lshlrev_b16_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +define void @shl_i16_v_s(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %b) { + %a = load i16, i16 addrspace(1)* %in + %result = shl i16 %a, %b + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_i16_v_compute_s: +; SI: v_lshlrev_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}} + +; VI: v_lshlrev_b16_e64 v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}} +define void @shl_i16_v_compute_s(i16 addrspace(1)* %out, i16 addrspace(1)* %in, i16 %b) { + %a = load i16, i16 addrspace(1)* %in + %b.add = add i16 %b, 3 + %result = shl i16 %a, %b.add + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_i16_computed_amount: +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 3, v{{[0-9]+}} +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, [[ADD]], v{{[0-9]+}} +define void @shl_i16_computed_amount(i16 addrspace(1)* %out, i16 addrspace(1)* %in) { + %tid = call i32 @llvm.r600.read.tidig.x() #0 + %gep = getelementptr inbounds i16, i16 addrspace(1)* %in, i32 %tid + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %b_ptr = getelementptr i16, i16 addrspace(1)* %gep, i16 1 + %a = load volatile i16, i16 addrspace(1)* %in + %b = load volatile i16, i16 addrspace(1)* %b_ptr + %b.add = add i16 %b, 3 + %result = shl i16 %a, %b.add + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_i16_i_s: +; GCN: s_lshl_b32 s{{[0-9]+}}, s{{[0-9]+}}, 12 +define void @shl_i16_i_s(i16 addrspace(1)* %out, i16 zeroext %a) { + %result = shl i16 %a, 12 + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_v2i16: +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) { + %tid = call i32 @llvm.r600.read.tidig.x() #0 + %gep = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %in, i32 %tid + %gep.out = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %out, i32 %tid + %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %gep, i16 1 + %a = load <2 x i16>, <2 x i16> addrspace(1)* %in + %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr + %result = shl <2 x i16> %a, %b + store <2 x i16> %result, <2 x i16> addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}shl_v4i16: +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_lshlrev_b16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @shl_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) { + %tid = call i32 @llvm.r600.read.tidig.x() #0 + %gep = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %in, i32 %tid + %gep.out = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %out, i32 %tid + %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %gep, i16 1 + %a = load <4 x i16>, <4 x i16> addrspace(1)* %gep + %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr + %result = shl <4 x i16> %a, %b + store <4 x i16> %result, <4 x i16> addrspace(1)* %gep.out + ret void +} + ;EG-LABEL: {{^}}shl_i64: ;EG: SUB_INT {{\*? *}}[[COMPSH:T[0-9]+\.[XYZW]]], {{literal.[xy]}}, [[SHIFT:T[0-9]+\.[XYZW]]] ;EG: LSHR {{\* *}}[[TEMP:T[0-9]+\.[XYZW]]], [[OPLO:T[0-9]+\.[XYZW]]], {{[[COMPSH]]|PV.[XYZW]}} @@ -65,16 +157,13 @@ define void @shl_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in ;EG-DAG: CNDE_INT {{\*? *}}[[RESLO:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW]}} ;EG-DAG: CNDE_INT {{\*? *}}[[RESHI:T[0-9]+\.[XYZW]]], {{T[0-9]+\.[XYZW], .*}}, 0.0 -;SI: {{^}}shl_i64: -;SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}} - -;VI: {{^}}shl_i64: -;VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}} - +; GCN-LABEL: {{^}}shl_i64: +; SI: v_lshl_b64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v[0-9]+}} +; VI: v_lshlrev_b64 {{v\[[0-9]+:[0-9]+\], v[0-9]+, v\[[0-9]+:[0-9]+\]}} define void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) { %b_ptr = getelementptr i64, i64 addrspace(1)* %in, i64 1 - %a = load i64, i64 addrspace(1) * %in - %b = load i64, i64 addrspace(1) * %b_ptr + %a = load i64, i64 addrspace(1)* %in + %b = load i64, i64 addrspace(1)* %b_ptr %result = shl i64 %a, %b store i64 %result, i64 addrspace(1)* %out ret void @@ -112,8 +201,8 @@ define void @shl_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %in) { define void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in) { %b_ptr = getelementptr <2 x i64>, <2 x i64> addrspace(1)* %in, i64 1 - %a = load <2 x i64>, <2 x i64> addrspace(1) * %in - %b = load <2 x i64>, <2 x i64> addrspace(1) * %b_ptr + %a = load <2 x i64>, <2 x i64> addrspace(1)* %in + %b = load <2 x i64>, <2 x i64> addrspace(1)* %b_ptr %result = shl <2 x i64> %a, %b store <2 x i64> %result, <2 x i64> addrspace(1)* %out ret void @@ -175,8 +264,8 @@ define void @shl_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> addrspace(1)* %in define void @shl_v4i64(<4 x i64> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) { %b_ptr = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i64 1 - %a = load <4 x i64>, <4 x i64> addrspace(1) * %in - %b = load <4 x i64>, <4 x i64> addrspace(1) * %b_ptr + %a = load <4 x i64>, <4 x i64> addrspace(1)* %in + %b = load <4 x i64>, <4 x i64> addrspace(1)* %b_ptr %result = shl <4 x i64> %a, %b store <4 x i64> %result, <4 x i64> addrspace(1)* %out ret void diff --git a/test/CodeGen/AMDGPU/shl_add_constant.ll b/test/CodeGen/AMDGPU/shl_add_constant.ll index 13254d0bcf74..9b5f9fed4d79 100644 --- a/test/CodeGen/AMDGPU/shl_add_constant.ll +++ b/test/CodeGen/AMDGPU/shl_add_constant.ll @@ -74,8 +74,8 @@ define void @test_add_shl_add_constant(i32 addrspace(1)* %out, i32 %x, i32 %y) # ; SI-DAG: s_load_dword [[Y:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xc ; SI: s_lshl_b32 [[SHL3:s[0-9]+]], [[X]], 3 ; SI: s_add_i32 [[TMP:s[0-9]+]], [[Y]], [[SHL3]] -; SI: s_add_i32 [[RESULT:s[0-9]+]], [[TMP]], 0x3d8 -; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[RESULT]] +; SI: s_addk_i32 [[TMP]], 0x3d8 +; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[TMP]] ; SI: buffer_store_dword [[VRESULT]] define void @test_add_shl_add_constant_inv(i32 addrspace(1)* %out, i32 %x, i32 %y) #0 { diff --git a/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll b/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll new file mode 100644 index 000000000000..ef616eb63801 --- /dev/null +++ b/test/CodeGen/AMDGPU/si-annotate-cf-noloop.ll @@ -0,0 +1,70 @@ +; RUN: opt -mtriple=amdgcn-- -S -structurizecfg -si-annotate-control-flow %s | FileCheck -check-prefix=OPT %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + + +; OPT-LABEL: @annotate_unreachable_noloop( +; OPT-NOT: call i1 @llvm.amdgcn.loop + +; GCN-LABEL: {{^}}annotate_unreachable_noloop: +; GCN: s_cbranch_vccnz +; GCN-NOT: s_endpgm +; GCN: .Lfunc_end0 +define void @annotate_unreachable_noloop(<4 x float> addrspace(1)* noalias nocapture readonly %arg) #0 { +bb: + %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() + br label %bb1 + +bb1: ; preds = %bb + %tmp2 = sext i32 %tmp to i64 + %tmp3 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %arg, i64 %tmp2 + %tmp4 = load <4 x float>, <4 x float> addrspace(1)* %tmp3, align 16 + br i1 undef, label %bb5, label %bb3 + +bb3: ; preds = %bb1 + %tmp6 = extractelement <4 x float> %tmp4, i32 2 + %tmp7 = fcmp olt float %tmp6, 0.000000e+00 + br i1 %tmp7, label %bb4, label %bb5 ; crash goes away if these are swapped + +bb4: ; preds = %bb3 + unreachable + +bb5: ; preds = %bb3, %bb1 + unreachable +} + + +; OPT-LABEL: @annotate_ret_noloop( +; OPT-NOT: call i1 @llvm.amdgcn.loop + +; GCN-LABEL: {{^}}annotate_ret_noloop: +; GCN: s_cbranch_scc1 +; GCN: s_endpgm +; GCN: .Lfunc_end1 +define void @annotate_ret_noloop(<4 x float> addrspace(1)* noalias nocapture readonly %arg) #0 { +bb: + %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() + br label %bb1 + +bb1: ; preds = %bb + %tmp2 = sext i32 %tmp to i64 + %tmp3 = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %arg, i64 %tmp2 + %tmp4 = load <4 x float>, <4 x float> addrspace(1)* %tmp3, align 16 + br i1 undef, label %bb5, label %bb3 + +bb3: ; preds = %bb1 + %tmp6 = extractelement <4 x float> %tmp4, i32 2 + %tmp7 = fcmp olt float %tmp6, 0.000000e+00 + br i1 %tmp7, label %bb4, label %bb5 ; crash goes away if these are swapped + +bb4: ; preds = %bb3 + ret void + +bb5: ; preds = %bb3, %bb1 + ret void +} + + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/si-annotate-cf.ll b/test/CodeGen/AMDGPU/si-annotate-cf.ll index 133fd480e599..175fa0e9f950 100644 --- a/test/CodeGen/AMDGPU/si-annotate-cf.ll +++ b/test/CodeGen/AMDGPU/si-annotate-cf.ll @@ -30,7 +30,7 @@ ENDIF: ; FIXME: This could be folded into the s_or_b64 instruction ; SI: s_mov_b64 [[ZERO:s\[[0-9]+:[0-9]+\]]], 0 ; SI: [[LOOP_LABEL:[A-Z0-9]+]] -; SI: v_cmp_ne_i32_e32 vcc, 0, v{{[0-9]+}} +; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}} ; SI_IF_BREAK instruction: ; SI: s_or_b64 [[BREAK:s\[[0-9]+:[0-9]+\]]], vcc, [[ZERO]] @@ -92,7 +92,7 @@ declare float @llvm.fabs.f32(float) nounwind readnone ; SI: s_cmp_gt_i32 ; SI-NEXT: s_cbranch_scc0 [[ENDPGM:BB[0-9]+_[0-9]+]] -; SI: s_cmp_gt_i32 +; SI: s_cmpk_gt_i32 ; SI-NEXT: s_cbranch_scc1 [[ENDPGM]] ; SI: [[INFLOOP:BB[0-9]+_[0-9]+]] diff --git a/test/CodeGen/AMDGPU/si-fix-sgpr-copies.mir b/test/CodeGen/AMDGPU/si-fix-sgpr-copies.mir new file mode 100644 index 000000000000..0c08deb13a8e --- /dev/null +++ b/test/CodeGen/AMDGPU/si-fix-sgpr-copies.mir @@ -0,0 +1,43 @@ +# RUN: llc -march=amdgcn -run-pass si-fix-sgpr-copies %s -o - | FileCheck %s -check-prefixes=GCN + +--- | + define void @phi_visit_order() { ret void } + +name: phi_visit_order +tracksRegLiveness: true +registers: + - { id: 0, class: sreg_32_xm0 } + - { id: 1, class: sreg_64 } + - { id: 2, class: sreg_32_xm0 } + - { id: 7, class: vgpr_32 } + - { id: 8, class: sreg_32_xm0 } + - { id: 9, class: vgpr_32 } + - { id: 10, class: sreg_64 } + - { id: 11, class: sreg_32_xm0 } + +body: | + ; GCN-LABEL: name: phi_visit_order + ; GCN: V_ADD_I32 + bb.0: + liveins: %vgpr0 + successors: %bb.1 + %7 = COPY %vgpr0 + %8 = S_MOV_B32 0 + + bb.1: + successors: %bb.1, %bb.2 + %0 = PHI %8, %bb.0, %0, %bb.1, %2, %bb.2 + %9 = V_MOV_B32_e32 9, implicit %exec + %10 = V_CMP_EQ_U32_e64 %7, %9, implicit %exec + %1 = SI_IF %10, %bb.2, implicit-def %exec, implicit-def %scc, implicit %exec + S_BRANCH %bb.1 + + bb.2: + successors: %bb.1 + SI_END_CF %1, implicit-def %exec, implicit-def %scc, implicit %exec + %11 = S_MOV_B32 1 + %2 = S_ADD_I32 %0, %11, implicit-def %scc + S_BRANCH %bb.1 + +... +--- diff --git a/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll b/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll index 98d1bb7cf9a2..0d1de6662f25 100644 --- a/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll +++ b/test/CodeGen/AMDGPU/si-instr-info-correct-implicit-operands.ll @@ -3,7 +3,7 @@ ; register operands in the correct order when modifying the opcode of an ; instruction to V_ADD_I32_e32. -; CHECK: %{{[0-9]+}} = V_ADD_I32_e32 killed %{{[0-9]+}}, killed %{{[0-9]+}}, implicit-def %vcc, implicit %exec +; CHECK: %{{[0-9]+}} = V_ADD_I32_e32 %{{[0-9]+}}, %{{[0-9]+}}, implicit-def %vcc, implicit %exec define void @test(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { entry: diff --git a/test/CodeGen/AMDGPU/si-literal-folding.ll b/test/CodeGen/AMDGPU/si-literal-folding.ll index d5030adc89be..b3f000c8ccd2 100644 --- a/test/CodeGen/AMDGPU/si-literal-folding.ll +++ b/test/CodeGen/AMDGPU/si-literal-folding.ll @@ -1,9 +1,8 @@ -; XFAIL: * -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck %s - -; CHECK-LABEL: {{^}}main: -; CHECK-NOT: v_mov_b32_e32 v{{[0-9]+}}, 0xbf4353f8 +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; GCN-LABEL: {{^}}main: +; GCN: v_mul_f32_e32 v{{[0-9]+}}, 0x3f4353f8, v{{[0-9]+}} +; GCN: v_mul_f32_e32 v{{[0-9]+}}, 0xbf4353f8, v{{[0-9]+}} define amdgpu_vs void @main(float) { main_body: %1 = fmul float %0, 0x3FE86A7F00000000 diff --git a/test/CodeGen/AMDGPU/si-lod-bias.ll b/test/CodeGen/AMDGPU/si-lod-bias.ll index 8df0a64a2b7c..bca34af3e270 100644 --- a/test/CodeGen/AMDGPU/si-lod-bias.ll +++ b/test/CodeGen/AMDGPU/si-lod-bias.ll @@ -49,4 +49,5 @@ declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float attributes #1 = { nounwind readnone } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !2} +!2 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll index ea506e6b3b3f..8d66df258e43 100644 --- a/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll +++ b/test/CodeGen/AMDGPU/si-lower-control-flow-unreachable-block.ll @@ -1,49 +1,56 @@ ; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator: -; GCN: v_cmp_eq_i32 +; GCN: v_cmp_eq_u32 ; GCN: s_and_saveexec_b64 ; GCN: s_xor_b64 -; GCN: s_branch BB0_1 +; GCN: ; mask branch [[RET:BB[0-9]+]] +; GCN: s_branch [[UNREACHABLE:BB[0-9]+_[0-9]+]] +; GCN: [[RET]] ; GCN: s_or_b64 exec, exec ; GCN: s_endpgm +; GCN: [[UNREACHABLE]]: ; GCN: ds_write_b32 ; GCN: s_waitcnt define void @lower_control_flow_unreachable_terminator() #0 { bb: %tmp15 = tail call i32 @llvm.amdgcn.workitem.id.y() %tmp63 = icmp eq i32 %tmp15, 32 - br i1 %tmp63, label %bb64, label %bb68 + br i1 %tmp63, label %unreachable, label %ret -bb64: +unreachable: store volatile i32 0, i32 addrspace(3)* undef, align 4 unreachable -bb68: +ret: ret void } ; GCN-LABEL: {{^}}lower_control_flow_unreachable_terminator_swap_block_order: -; GCN: v_cmp_eq_i32 +; GCN: v_cmp_eq_u32 ; GCN: s_and_saveexec_b64 ; GCN: s_xor_b64 -; GCN: s_endpgm +; GCN: ; mask branch [[UNREACHABLE:BB[0-9]+_[0-9]+]] -; GCN: s_or_b64 exec, exec +; GCN-NEXT: ; %ret +; GCN-NEXT: s_endpgm + +; GCN-NEXT: [[UNREACHABLE]]: +; GCN-NEXT: s_or_b64 exec, exec ; GCN: ds_write_b32 ; GCN: s_waitcnt define void @lower_control_flow_unreachable_terminator_swap_block_order() #0 { bb: %tmp15 = tail call i32 @llvm.amdgcn.workitem.id.y() %tmp63 = icmp eq i32 %tmp15, 32 - br i1 %tmp63, label %bb68, label %bb64 + br i1 %tmp63, label %ret, label %unreachable -bb68: +ret: ret void -bb64: +unreachable: store volatile i32 0, i32 addrspace(3)* undef, align 4 unreachable } diff --git a/test/CodeGen/AMDGPU/si-scheduler.ll b/test/CodeGen/AMDGPU/si-scheduler.ll index 5520fe61d867..9374ef3cd907 100644 --- a/test/CodeGen/AMDGPU/si-scheduler.ll +++ b/test/CodeGen/AMDGPU/si-scheduler.ll @@ -57,4 +57,5 @@ declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float attributes #1 = { nounwind readnone } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !2} +!2 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/si-sgpr-spill.ll b/test/CodeGen/AMDGPU/si-sgpr-spill.ll index 63b1b71a8cb7..f035da83d18c 100644 --- a/test/CodeGen/AMDGPU/si-sgpr-spill.ll +++ b/test/CodeGen/AMDGPU/si-sgpr-spill.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck %s ; These tests check that the compiler won't crash when it needs to spill ; SGPRs. @@ -1619,4 +1619,5 @@ attributes #2 = { nounwind readnone } attributes #3 = { nounwind readonly } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !2} +!2 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll b/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll index 5171406469ab..e9a3a7f3fcff 100644 --- a/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll +++ b/test/CodeGen/AMDGPU/si-spill-sgpr-stack.ll @@ -1,14 +1,29 @@ -; RUN: llc -march=amdgcn -mcpu=fiji < %s | FileCheck %s +; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SGPR %s +; RUN: llc -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=1 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SMEM %s ; Make sure this doesn't crash. -; CHECK: {{^}}test: +; ALL-LABEL: {{^}}test: +; ALL: s_mov_b32 s[[LO:[0-9]+]], SCRATCH_RSRC_DWORD0 +; ALL: s_mov_b32 s[[OFF:[0-9]+]], s3 +; ALL: s_mov_b32 s[[HI:[0-9]+]], 0xe80000 + ; Make sure we are handling hazards correctly. -; CHECK: buffer_load_dword [[VHI:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:12 -; CHECK-NEXT: s_waitcnt vmcnt(0) -; CHECK-NEXT: v_readfirstlane_b32 s[[HI:[0-9]+]], [[VHI]] -; CHECK-NEXT: s_nop 4 -; CHECK-NEXT: buffer_store_dword v0, off, s[0:[[HI]]{{\]}}, 0 -; CHECK: s_endpgm +; SGPR: buffer_load_dword [[VHI:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], s{{[0-9]+}} offset:12 +; SGPR-NEXT: s_waitcnt vmcnt(0) +; SGPR-NEXT: v_readfirstlane_b32 s[[HI:[0-9]+]], [[VHI]] +; SGPR-NEXT: s_nop 4 +; SGPR-NEXT: buffer_store_dword v0, off, s[0:[[HI]]{{\]}}, 0 + +; Make sure scratch wave offset register is correctly incremented and +; then restored. +; SMEM: s_mov_b32 m0, s[[OFF]]{{$}} +; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[LO]]:[[HI]]], m0 ; 16-byte Folded Spill + +; SMEM: s_mov_b32 m0, s[[OFF]]{{$}} +; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[LO]]:[[HI]]], m0 ; 16-byte Folded Reload + +; SMEM: s_dcache_wb +; ALL: s_endpgm define void @test(i32 addrspace(1)* %out, i32 %in) { call void asm sideeffect "", "~{SGPR0_SGPR1_SGPR2_SGPR3_SGPR4_SGPR5_SGPR6_SGPR7}" () call void asm sideeffect "", "~{SGPR8_SGPR9_SGPR10_SGPR11_SGPR12_SGPR13_SGPR14_SGPR15}" () diff --git a/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll index 0e9618523e32..4beefb047f22 100644 --- a/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll +++ b/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll @@ -3,6 +3,7 @@ declare void @llvm.SI.tbuffer.store.i32(<16 x i8>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) declare void @llvm.SI.tbuffer.store.v4i32(<16 x i8>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) declare void @llvm.amdgcn.s.barrier() #1 +declare i32 @llvm.amdgcn.workitem.id.x() #2 @stored_lds_ptr = addrspace(3) global i32 addrspace(3)* undef, align 4 @@ -70,9 +71,9 @@ define void @no_reorder_barrier_local_load_global_store_local_load(i32 addrspace } ; FUNC-LABEL: @reorder_constant_load_global_store_constant_load -; CI-DAG: buffer_store_dword ; CI-DAG: v_readfirstlane_b32 s[[PTR_LO:[0-9]+]], v{{[0-9]+}} ; CI: v_readfirstlane_b32 s[[PTR_HI:[0-9]+]], v{{[0-9]+}} +; CI: buffer_store_dword ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x1 ; CI-DAG: s_load_dword s{{[0-9]+}}, s{{\[}}[[PTR_LO]]:[[PTR_HI]]{{\]}}, 0x3 ; CI: buffer_store_dword @@ -136,9 +137,9 @@ define void @reorder_smrd_load_local_store_smrd_load(i32 addrspace(1)* %out, i32 } ; FUNC-LABEL: @reorder_global_load_local_store_global_load +; CI: ds_write_b32 ; CI: buffer_load_dword ; CI: buffer_load_dword -; CI: ds_write_b32 ; CI: buffer_store_dword define void @reorder_global_load_local_store_global_load(i32 addrspace(1)* %out, i32 addrspace(3)* %lptr, i32 addrspace(1)* %ptr0) #0 { %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i64 1 @@ -156,7 +157,8 @@ define void @reorder_global_load_local_store_global_load(i32 addrspace(1)* %out, ; FUNC-LABEL: @reorder_local_offsets ; CI: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:100 offset1:102 -; CI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:400 +; CI: ds_write2_b32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}} offset0:3 offset1:100 +; CI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:12 ; CI: ds_write_b32 {{v[0-9]+}}, {{v[0-9]+}} offset:408 ; CI: buffer_store_dword ; CI: s_endpgm @@ -180,11 +182,11 @@ define void @reorder_local_offsets(i32 addrspace(1)* nocapture %out, i32 addrspa ; FUNC-LABEL: @reorder_global_offsets ; CI: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400 -; CI: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12 ; CI: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408 -; CI: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12 +; CI: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12 ; CI: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:400 ; CI: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:408 +; CI: buffer_load_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 offset:12 ; CI: s_endpgm define void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrspace(1)* noalias nocapture readnone %gptr, i32 addrspace(1)* noalias nocapture %ptr0) #0 { %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3 @@ -204,6 +206,46 @@ define void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrsp ret void } +; FUNC-LABEL: {{^}}reorder_global_offsets_addr64_soffset0: +; GCN: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:12{{$}} +; GCN-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:28{{$}} +; GCN-NEXT: buffer_load_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:44{{$}} + +; GCN: v_mov_b32 +; GCN: v_mov_b32 + +; GCN: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64{{$}} +; GCN-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:20{{$}} + +; GCN: v_add_i32 +; GCN: v_add_i32 + +; GCN: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:36{{$}} +; GCN-NEXT: buffer_store_dword v{{[0-9]+}}, v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}} 0 addr64 offset:52{{$}} +define void @reorder_global_offsets_addr64_soffset0(i32 addrspace(1)* noalias nocapture %ptr.base) #0 { + %id = call i32 @llvm.amdgcn.workitem.id.x() + %id.ext = sext i32 %id to i64 + + %ptr0 = getelementptr inbounds i32, i32 addrspace(1)* %ptr.base, i64 %id.ext + %ptr1 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 3 + %ptr2 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 5 + %ptr3 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 7 + %ptr4 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 9 + %ptr5 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 11 + %ptr6 = getelementptr inbounds i32, i32 addrspace(1)* %ptr0, i32 13 + + store i32 789, i32 addrspace(1)* %ptr0, align 4 + %tmp1 = load i32, i32 addrspace(1)* %ptr1, align 4 + store i32 123, i32 addrspace(1)* %ptr2, align 4 + %tmp2 = load i32, i32 addrspace(1)* %ptr3, align 4 + %add.0 = add nsw i32 %tmp1, %tmp2 + store i32 %add.0, i32 addrspace(1)* %ptr4, align 4 + %tmp3 = load i32, i32 addrspace(1)* %ptr5, align 4 + %add.1 = add nsw i32 %add.0, %tmp3 + store i32 %add.1, i32 addrspace(1)* %ptr6, align 4 + ret void +} + ; XFUNC-LABEL: @reorder_local_load_tbuffer_store_local_load ; XCI: ds_read_b32 {{v[0-9]+}}, {{v[0-9]+}}, 0x4 ; XCI: TBUFFER_STORE_FORMAT @@ -231,3 +273,4 @@ define void @reorder_global_offsets(i32 addrspace(1)* nocapture %out, i32 addrsp attributes #0 = { nounwind } attributes #1 = { nounwind convergent } +attributes #2 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/sign_extend.ll b/test/CodeGen/AMDGPU/sign_extend.ll index 30e6bd1e78f2..05938170eba1 100644 --- a/test/CodeGen/AMDGPU/sign_extend.ll +++ b/test/CodeGen/AMDGPU/sign_extend.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s ; GCN-LABEL: {{^}}s_sext_i1_to_i32: ; GCN: v_cndmask_b32_e64 @@ -55,22 +55,72 @@ define void @v_sext_i32_to_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %in) no } ; GCN-LABEL: {{^}}s_sext_i16_to_i64: -; GCN: s_endpgm +; GCN: s_bfe_i64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x100000 define void @s_sext_i16_to_i64(i64 addrspace(1)* %out, i16 %a) nounwind { %sext = sext i16 %a to i64 store i64 %sext, i64 addrspace(1)* %out, align 8 ret void } +; GCN-LABEL: {{^}}s_sext_i1_to_i16: +; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1 +; GCN-NEXT: buffer_store_short [[RESULT]] +define void @s_sext_i1_to_i16(i16 addrspace(1)* %out, i32 %a, i32 %b) nounwind { + %cmp = icmp eq i32 %a, %b + %sext = sext i1 %cmp to i16 + store i16 %sext, i16 addrspace(1)* %out + ret void +} + +; This purpose of this test is to make sure the i16 = sign_extend i1 node +; makes it all the way throught the legalizer/optimizer to make sure +; we select this correctly. In the s_sext_i1_to_i16, the sign_extend node +; is optimized to a select very early. +; GCN-LABEL: {{^}}s_sext_i1_to_i16_with_and: +; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1 +; GCN-NEXT: buffer_store_short [[RESULT]] +define void @s_sext_i1_to_i16_with_and(i16 addrspace(1)* %out, i32 %a, i32 %b, i32 %c, i32 %d) nounwind { + %cmp0 = icmp eq i32 %a, %b + %cmp1 = icmp eq i32 %c, %d + %cmp = and i1 %cmp0, %cmp1 + %sext = sext i1 %cmp to i16 + store i16 %sext, i16 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}v_sext_i1_to_i16_with_and: +; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1 +; GCN-NEXT: buffer_store_short [[RESULT]] +define void @v_sext_i1_to_i16_with_and(i16 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) nounwind { + %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #1 + %cmp0 = icmp eq i32 %a, %tid + %cmp1 = icmp eq i32 %b, %c + %cmp = and i1 %cmp0, %cmp1 + %sext = sext i1 %cmp to i16 + store i16 %sext, i16 addrspace(1)* %out + ret void +} + ; GCN-LABEL: {{^}}s_sext_v4i8_to_v4i32: ; GCN: s_load_dword [[VAL:s[0-9]+]] -; GCN-DAG: s_sext_i32_i8 [[EXT0:s[0-9]+]], [[VAL]] -; GCN-DAG: s_bfe_i32 [[EXT1:s[0-9]+]], [[VAL]], 0x80008 ; GCN-DAG: s_bfe_i32 [[EXT2:s[0-9]+]], [[VAL]], 0x80010 ; GCN-DAG: s_ashr_i32 [[EXT3:s[0-9]+]], [[VAL]], 24 +; SI-DAG: s_bfe_i32 [[EXT1:s[0-9]+]], [[VAL]], 0x80008 +; GCN-DAG: s_sext_i32_i8 [[EXT0:s[0-9]+]], [[VAL]] + +; FIXME: We end up with a v_bfe instruction, because the i16 srl +; gets selected to a v_lshrrev_b16 instructions, so the input to +; the bfe is a vector registers. To fix this we need to be able to +; optimize: +; t29: i16 = truncate t10 +; t55: i16 = srl t29, Constant:i32<8> +; t63: i32 = any_extend t55 +; t64: i32 = sign_extend_inreg t63, ValueType:ch:i8 + +; VI-DAG: v_bfe_i32 [[VEXT1:v[0-9]+]], v{{[0-9]+}}, 0, 8 ; GCN-DAG: v_mov_b32_e32 [[VEXT0:v[0-9]+]], [[EXT0]] -; GCN-DAG: v_mov_b32_e32 [[VEXT1:v[0-9]+]], [[EXT1]] +; SI-DAG: v_mov_b32_e32 [[VEXT1:v[0-9]+]], [[EXT1]] ; GCN-DAG: v_mov_b32_e32 [[VEXT2:v[0-9]+]], [[EXT2]] ; GCN-DAG: v_mov_b32_e32 [[VEXT3:v[0-9]+]], [[EXT3]] @@ -96,10 +146,17 @@ define void @s_sext_v4i8_to_v4i32(i32 addrspace(1)* %out, i32 %a) nounwind { ; GCN-LABEL: {{^}}v_sext_v4i8_to_v4i32: ; GCN: buffer_load_dword [[VAL:v[0-9]+]] -; GCN-DAG: v_bfe_i32 [[EXT0:v[0-9]+]], [[VAL]], 0, 8 -; GCN-DAG: v_bfe_i32 [[EXT1:v[0-9]+]], [[VAL]], 8, 8 -; GCN-DAG: v_bfe_i32 [[EXT2:v[0-9]+]], [[VAL]], 16, 8 +; FIXME: need to optimize same sequence as above test to avoid +; this shift. +; VI-DAG: v_lshrrev_b16_e32 [[SH16:v[0-9]+]], 8, [[VAL]] ; GCN-DAG: v_ashrrev_i32_e32 [[EXT3:v[0-9]+]], 24, [[VAL]] +; VI-DAG: v_bfe_i32 [[EXT0:v[0-9]+]], [[VAL]], 0, 8 +; VI-DAG: v_bfe_i32 [[EXT2:v[0-9]+]], [[VAL]], 16, 8 +; VI-DAG: v_bfe_i32 [[EXT1:v[0-9]+]], [[SH16]], 0, 8 + +; SI-DAG: v_bfe_i32 [[EXT2:v[0-9]+]], [[VAL]], 16, 8 +; SI-DAG: v_bfe_i32 [[EXT1:v[0-9]+]], [[VAL]], 8, 8 +; SI: v_bfe_i32 [[EXT0:v[0-9]+]], [[VAL]], 0, 8 ; GCN: buffer_store_dword [[EXT0]] ; GCN: buffer_store_dword [[EXT1]] @@ -163,3 +220,7 @@ define void @v_sext_v4i16_to_v4i32(i32 addrspace(1)* %out, i64 addrspace(1)* %in store volatile i32 %elt3, i32 addrspace(1)* %out ret void } + +declare i32 @llvm.amdgcn.workitem.id.x() #1 + +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/simplify-demanded-bits-build-pair.ll b/test/CodeGen/AMDGPU/simplify-demanded-bits-build-pair.ll deleted file mode 100644 index dffee70b6b02..000000000000 --- a/test/CodeGen/AMDGPU/simplify-demanded-bits-build-pair.ll +++ /dev/null @@ -1,39 +0,0 @@ -; XFAIL: * -; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI -mattr=-promote-alloca < %s | FileCheck -check-prefix=SI %s -; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga -mattr=-promote-alloca < %s | FileCheck -check-prefix=SI %s - -; 64-bit select was originally lowered with a build_pair, and this -; could be simplified to 1 cndmask instead of 2, but that broken when -; it started being implemented with a v2i32 build_vector and -; bitcasting. -define void @trunc_select_i64(i32 addrspace(1)* %out, i64 %a, i64 %b, i32 %c) { - %cmp = icmp eq i32 %c, 0 - %select = select i1 %cmp, i64 %a, i64 %b - %trunc = trunc i64 %select to i32 - store i32 %trunc, i32 addrspace(1)* %out, align 4 - ret void -} - -; FIXME: Fix truncating store for local memory -; SI-LABEL: {{^}}trunc_load_alloca_i64: -; SI: v_movrels_b32 -; SI-NOT: v_movrels_b32 -; SI: s_endpgm -define void @trunc_load_alloca_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) { - %idx = add i32 %a, %b - %alloca = alloca i64, i32 4 - %gep0 = getelementptr i64, i64* %alloca, i64 0 - %gep1 = getelementptr i64, i64* %alloca, i64 1 - %gep2 = getelementptr i64, i64* %alloca, i64 2 - %gep3 = getelementptr i64, i64* %alloca, i64 3 - store i64 24, i64* %gep0, align 8 - store i64 9334, i64* %gep1, align 8 - store i64 3935, i64* %gep2, align 8 - store i64 9342, i64* %gep3, align 8 - %gep = getelementptr i64, i64* %alloca, i32 %idx - %load = load i64, i64* %gep, align 8 - %mask = and i64 %load, 4294967296 - %add = add i64 %mask, -1 - store i64 %add, i64 addrspace(1)* %out, align 4 - ret void -} diff --git a/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/test/CodeGen/AMDGPU/sint_to_fp.f64.ll index 7c58f2d906d4..68dc3c6ccd24 100644 --- a/test/CodeGen/AMDGPU/sint_to_fp.f64.ll +++ b/test/CodeGen/AMDGPU/sint_to_fp.f64.ll @@ -14,7 +14,7 @@ define void @sint_to_fp_i32_to_f64(double addrspace(1)* %out, i32 %in) { ; uses an SGPR (implicit vcc). ; SI-LABEL: {{^}}sint_to_fp_i1_f64: -; SI-DAG: v_cmp_eq_i32_e64 vcc, +; SI-DAG: v_cmp_eq_u32_e64 vcc, ; SI-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}} ; SI-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} ; SI: buffer_store_dwordx2 v{{\[}}[[ZERO]]:[[SEL]]{{\]}} diff --git a/test/CodeGen/AMDGPU/sint_to_fp.i64.ll b/test/CodeGen/AMDGPU/sint_to_fp.i64.ll index 16eae1899ec0..b0ae1c2969f9 100644 --- a/test/CodeGen/AMDGPU/sint_to_fp.i64.ll +++ b/test/CodeGen/AMDGPU/sint_to_fp.i64.ll @@ -3,6 +3,41 @@ ; FIXME: This should be merged with sint_to_fp.ll, but s_sint_to_fp_v2i64 crashes on r600 +; FUNC-LABEL: {{^}}s_sint_to_fp_i64_to_f16: +define void @s_sint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 %in) #0 { + %result = sitofp i64 %in to half + store half %result, half addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_sint_to_fp_i64_to_f16: +; GCN: {{buffer|flat}}_load_dwordx2 + +; GCN: v_ashrrev_i32_e32 v{{[0-9]+}}, 31, v{{[0-9]+}} +; GCN: v_xor_b32 + +; GCN: v_ffbh_u32 +; GCN: v_ffbh_u32 +; GCN: v_cndmask +; GCN: v_cndmask + +; GCN-DAG: v_cmp_eq_u64 +; GCN-DAG: v_cmp_lt_u64 + +; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} +; GCN: v_cndmask_b32_e{{32|64}} [[SIGN_SEL:v[0-9]+]], +; GCN: v_cvt_f16_f32_e32 [[SIGN_SEL_F16:v[0-9]+]], [[SIGN_SEL]] +; GCN: {{buffer|flat}}_store_short {{.*}}[[SIGN_SEL_F16]] +define void @v_sint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid + %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid + %val = load i64, i64 addrspace(1)* %in.gep + %result = sitofp i64 %val to half + store half %result, half addrspace(1)* %out.gep + ret void +} + ; FUNC-LABEL: {{^}}s_sint_to_fp_i64_to_f32: define void @s_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 %in) #0 { %result = sitofp i64 %in to float @@ -21,7 +56,7 @@ define void @s_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 %in) #0 { ; GCN: v_cndmask ; GCN: v_cndmask -; GCN-DAG: v_cmp_eq_i64 +; GCN-DAG: v_cmp_eq_u64 ; GCN-DAG: v_cmp_lt_u64 ; GCN: v_xor_b32_e32 v{{[0-9]+}}, 0x80000000, v{{[0-9]+}} @@ -37,15 +72,16 @@ define void @v_sint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* ret void } -; FUNC-LABEL: {{^}}s_sint_to_fp_v2i64: -define void @s_sint_to_fp_v2i64(<2 x float> addrspace(1)* %out, <2 x i64> %in) #0{ +; FUNC-LABEL: {{^}}s_sint_to_fp_v2i64_to_v2f32: +; GCN-NOT: v_and_b32_e32 v{{[0-9]+}}, -1, +define void @s_sint_to_fp_v2i64_to_v2f32(<2 x float> addrspace(1)* %out, <2 x i64> %in) #0{ %result = sitofp <2 x i64> %in to <2 x float> store <2 x float> %result, <2 x float> addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}v_sint_to_fp_v4i64: -define void @v_sint_to_fp_v4i64(<4 x float> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { +; FUNC-LABEL: {{^}}v_sint_to_fp_v4i64_to_v4f32: +define void @v_sint_to_fp_v4i64_to_v4f32(<4 x float> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { %tid = call i32 @llvm.amdgcn.workitem.id.x() %in.gep = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i32 %tid %out.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %out, i32 %tid @@ -55,6 +91,25 @@ define void @v_sint_to_fp_v4i64(<4 x float> addrspace(1)* %out, <4 x i64> addrsp ret void } +; FUNC-LABEL: {{^}}s_sint_to_fp_v2i64_to_v2f16: +; GCN-NOT: v_and_b32_e32 v{{[0-9]+}}, -1, +define void @s_sint_to_fp_v2i64_to_v2f16(<2 x half> addrspace(1)* %out, <2 x i64> %in) #0{ + %result = sitofp <2 x i64> %in to <2 x half> + store <2 x half> %result, <2 x half> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_sint_to_fp_v4i64_to_v4f16: +define void @v_sint_to_fp_v4i64_to_v4f16(<4 x half> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %in.gep = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i32 %tid + %out.gep = getelementptr <4 x half>, <4 x half> addrspace(1)* %out, i32 %tid + %value = load <4 x i64>, <4 x i64> addrspace(1)* %in.gep + %result = sitofp <4 x i64> %value to <4 x half> + store <4 x half> %result, <4 x half> addrspace(1)* %out.gep + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #1 attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/sint_to_fp.ll b/test/CodeGen/AMDGPU/sint_to_fp.ll index 75ffdd2cc85a..bae9bead46ab 100644 --- a/test/CodeGen/AMDGPU/sint_to_fp.ll +++ b/test/CodeGen/AMDGPU/sint_to_fp.ll @@ -77,7 +77,7 @@ define void @v_sint_to_fp_v4i32(<4 x float> addrspace(1)* %out, <4 x i32> addrsp } ; FUNC-LABEL: {{^}}s_sint_to_fp_i1_f32: -; SI: v_cmp_eq_i32_e64 [[CMP:s\[[0-9]+:[0-9]\]]], +; SI: v_cmp_eq_u32_e64 [[CMP:s\[[0-9]+:[0-9]\]]], ; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]] ; SI: buffer_store_dword [[RESULT]], ; SI: s_endpgm @@ -101,7 +101,7 @@ define void @s_sint_to_fp_i1_f32_load(float addrspace(1)* %out, i1 %in) #0 { ; FUNC-LABEL: {{^}}v_sint_to_fp_i1_f32_load: ; SI: {{buffer|flat}}_load_ubyte ; SI: v_and_b32_e32 {{v[0-9]+}}, 1, {{v[0-9]+}} -; SI: v_cmp_eq_i32 +; SI: v_cmp_eq_u32 ; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, -1.0 ; SI: {{buffer|flat}}_store_dword {{.*}}[[RESULT]] ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/sitofp.f16.ll b/test/CodeGen/AMDGPU/sitofp.f16.ll new file mode 100644 index 000000000000..0910d7718440 --- /dev/null +++ b/test/CodeGen/AMDGPU/sitofp.f16.ll @@ -0,0 +1,79 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}sitofp_i16_to_f16 +; GCN: buffer_load_{{sshort|ushort}} v[[A_I16:[0-9]+]] +; GCN: v_cvt_f32_i32_e32 v[[A_F32:[0-9]+]], v[[A_I16]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @sitofp_i16_to_f16( + half addrspace(1)* %r, + i16 addrspace(1)* %a) { +entry: + %a.val = load i16, i16 addrspace(1)* %a + %r.val = sitofp i16 %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}sitofp_i32_to_f16 +; GCN: buffer_load_dword v[[A_I32:[0-9]+]] +; GCN: v_cvt_f32_i32_e32 v[[A_I16:[0-9]+]], v[[A_I32]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_I16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @sitofp_i32_to_f16( + half addrspace(1)* %r, + i32 addrspace(1)* %a) { +entry: + %a.val = load i32, i32 addrspace(1)* %a + %r.val = sitofp i32 %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; f16 = sitofp i64 is in sint_to_fp.i64.ll + +; GCN-LABEL: {{^}}sitofp_v2i16_to_v2f16 +; GCN: buffer_load_dword +; GCN: v_cvt_f32_i32_e32 +; GCN: v_cvt_f32_i32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN-DAG: v_lshlrev_b32_e32 +; GCN-DAG: v_or_b32_e32 +; GCN: buffer_store_dword +; GCN: s_endpgm +define void @sitofp_v2i16_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x i16> addrspace(1)* %a) { +entry: + %a.val = load <2 x i16>, <2 x i16> addrspace(1)* %a + %r.val = sitofp <2 x i16> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}sitofp_v2i32_to_v2f16 +; GCN: buffer_load_dwordx2 +; GCN: v_cvt_f32_i32_e32 +; GCN: v_cvt_f32_i32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN-DAG: v_and_b32_e32 +; GCN-DAG: v_lshlrev_b32_e32 +; GCN-DAG: v_or_b32_e32 +; GCN: buffer_store_dword +; GCN: s_endpgm +define void @sitofp_v2i32_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x i32> addrspace(1)* %a) { +entry: + %a.val = load <2 x i32>, <2 x i32> addrspace(1)* %a + %r.val = sitofp <2 x i32> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; v2f16 = sitofp v2i64 is in sint_to_fp.i64.ll diff --git a/test/CodeGen/AMDGPU/skip-if-dead.ll b/test/CodeGen/AMDGPU/skip-if-dead.ll index 4ba4ac76a280..60cee7a3499e 100644 --- a/test/CodeGen/AMDGPU/skip-if-dead.ll +++ b/test/CodeGen/AMDGPU/skip-if-dead.ll @@ -87,7 +87,7 @@ define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 { ; FIXME: why does the skip depend on the asm length in the same block? ; CHECK-LABEL: {{^}}test_kill_control_flow: -; CHECK: s_cmp_lg_i32 s{{[0-9]+}}, 0 +; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0 ; CHECK: s_cbranch_scc1 [[RETURN_BB:BB[0-9]+_[0-9]+]] ; CHECK-NEXT: ; BB#1: @@ -105,8 +105,8 @@ define amdgpu_ps void @test_kill_depth_var_x2_instructions(float %x) #0 { ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7 ; CHECK-NEXT: s_cbranch_execnz [[SPLIT_BB:BB[0-9]+_[0-9]+]] -; CHECK-NEXT: ; BB#3: -; CHECK-NEXT: exp 0, 9, 0, 1, 1, v0, v0, v0, v0 +; CHECK-NEXT: ; BB#2: +; CHECK-NEXT: exp null off, off, off, off done vm ; CHECK-NEXT: s_endpgm ; CHECK-NEXT: {{^}}[[SPLIT_BB]]: @@ -137,7 +137,8 @@ exit: } ; CHECK-LABEL: {{^}}test_kill_control_flow_remainder: -; CHECK: s_cmp_lg_i32 s{{[0-9]+}}, 0 +; CHECK: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; CHECK-NEXT: v_mov_b32_e32 v{{[0-9]+}}, 0 ; CHECK-NEXT: s_cbranch_scc1 [[RETURN_BB:BB[0-9]+_[0-9]+]] ; CHECK-NEXT: ; BB#1: ; %bb @@ -156,8 +157,8 @@ exit: ; CHECK: v_cmpx_le_f32_e32 vcc, 0, v7 ; CHECK-NEXT: s_cbranch_execnz [[SPLIT_BB:BB[0-9]+_[0-9]+]] -; CHECK-NEXT: ; BB#4: -; CHECK-NEXT: exp 0, 9, 0, 1, 1, v0, v0, v0, v0 +; CHECK-NEXT: ; BB#2: +; CHECK-NEXT: exp null off, off, off, off done vm ; CHECK-NEXT: s_endpgm ; CHECK-NEXT: {{^}}[[SPLIT_BB]]: @@ -199,11 +200,14 @@ exit: } ; CHECK-LABEL: {{^}}test_kill_divergent_loop: -; CHECK: v_cmp_eq_i32_e32 vcc, 0, v0 +; CHECK: v_cmp_eq_u32_e32 vcc, 0, v0 ; CHECK-NEXT: s_and_saveexec_b64 [[SAVEEXEC:s\[[0-9]+:[0-9]+\]]], vcc ; CHECK-NEXT: s_xor_b64 [[SAVEEXEC]], exec, [[SAVEEXEC]] -; CHECK-NEXT: s_cbranch_execz [[EXIT:BB[0-9]+_[0-9]+]] -; CHECK-NEXT: ; mask branch [[EXIT]] +; CHECK-NEXT: ; mask branch [[EXIT:BB[0-9]+_[0-9]+]] +; CHECK-NEXT: s_cbranch_execz [[EXIT]] + +; CHECK: {{BB[0-9]+_[0-9]+}}: ; %bb.preheader +; CHECK: s_mov_b32 ; CHECK: [[LOOP_BB:BB[0-9]+_[0-9]+]]: @@ -213,7 +217,7 @@ exit: ; CHECK-NEXT: ; BB#3: ; CHECK: buffer_load_dword [[LOAD:v[0-9]+]] -; CHECK: v_cmp_eq_i32_e32 vcc, 0, [[LOAD]] +; CHECK: v_cmp_eq_u32_e32 vcc, 0, [[LOAD]] ; CHECK-NEXT: s_and_b64 vcc, exec, vcc ; CHECK-NEXT: s_cbranch_vccnz [[LOOP_BB]] @@ -259,15 +263,13 @@ exit: ; CHECK-NEXT: s_endpgm ; CHECK: [[KILLBB:BB[0-9]+_[0-9]+]]: -; CHECK: s_and_b64 vcc, exec, -; CHECK-NEXT: s_cbranch_vccz [[PHIBB:BB[0-9]+_[0-9]+]] +; CHECK-NEXT: s_cbranch_scc0 [[PHIBB:BB[0-9]+_[0-9]+]] ; CHECK: [[PHIBB]]: ; CHECK: v_cmp_eq_f32_e32 vcc, 0, [[PHIREG]] -; CHECK: s_and_b64 vcc, exec, vcc -; CHECK: s_cbranch_vccz [[ENDBB:BB[0-9]+_[0-9]+]] +; CHECK-NEXT: s_cbranch_vccz [[ENDBB:BB[0-9]+_[0-9]+]] -; CHECK: ; BB#3: ; %bb10 +; CHECK: ; %bb10 ; CHECK: v_mov_b32_e32 v{{[0-9]+}}, 9 ; CHECK: buffer_store_dword @@ -299,19 +301,15 @@ end: } ; CHECK-LABEL: {{^}}no_skip_no_successors: -; CHECK: v_cmp_nle_f32 -; CHECK: s_and_b64 vcc, exec, -; CHECK: s_cbranch_vccz [[SKIPKILL:BB[0-9]+_[0-9]+]] +; CHECK: v_cmp_nge_f32 +; CHECK-NEXT: s_cbranch_vccz [[SKIPKILL:BB[0-9]+_[0-9]+]] -; CHECK: ; BB#3: ; %bb6 +; CHECK: ; %bb6 ; CHECK: s_mov_b64 exec, 0 ; CHECK: [[SKIPKILL]]: -; CHECK: v_cmp_nge_f32 -; CHECK: s_and_b64 vcc, exec, vcc -; CHECK: s_cbranch_vccz [[UNREACHABLE:BB[0-9]+_[0-9]+]] - -; CHECK: [[UNREACHABLE]]: +; CHECK: v_cmp_nge_f32_e32 vcc +; CHECK-NEXT: BB#3: ; %bb5 ; CHECK-NEXT: .Lfunc_end{{[0-9]+}} define amdgpu_ps void @no_skip_no_successors(float inreg %arg, float inreg %arg1) #0 { bb: @@ -353,7 +351,7 @@ bb7: ; preds = %bb4 ; CHECK: mask branch [[END:BB[0-9]+_[0-9]+]] ; CHECK-NOT: branch -; CHECK: ; BB#3: ; %bb8 +; CHECK: BB{{[0-9]+_[0-9]+}}: ; %bb8 ; CHECK: buffer_store_dword ; CHECK: [[END]]: @@ -387,4 +385,4 @@ declare <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32>, <8 x i32>, <4 x i32 declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) nounwind attributes #0 = { nounwind } -attributes #1 = { nounwind readnone }
\ No newline at end of file +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/sminmax.ll b/test/CodeGen/AMDGPU/sminmax.ll index 560d5597baa9..83465206e066 100644 --- a/test/CodeGen/AMDGPU/sminmax.ll +++ b/test/CodeGen/AMDGPU/sminmax.ll @@ -32,6 +32,20 @@ define void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind ret void } +; GCN-LABEL: {{^}}v_abs_i32_repeat_user: +; GCN: v_sub_i32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]] +; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[NEG]], [[SRC]] +; GCN: v_mul_lo_i32 v{{[0-9]+}}, [[MAX]], [[MAX]] +define void @v_abs_i32_repeat_user(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind { + %val = load i32, i32 addrspace(1)* %src, align 4 + %neg = sub i32 0, %val + %cond = icmp sgt i32 %val, %neg + %res = select i1 %cond, i32 %val, i32 %neg + %mul = mul i32 %res, %res + store i32 %mul, i32 addrspace(1)* %out, align 4 + ret void +} + ; FUNC-LABEL: {{^}}s_abs_v2i32: ; GCN: s_abs_i32 ; GCN: s_abs_i32 diff --git a/test/CodeGen/AMDGPU/smrd-vccz-bug.ll b/test/CodeGen/AMDGPU/smrd-vccz-bug.ll index ddac8a006c86..f82ac041969c 100644 --- a/test/CodeGen/AMDGPU/smrd-vccz-bug.ll +++ b/test/CodeGen/AMDGPU/smrd-vccz-bug.ll @@ -4,8 +4,7 @@ ; GCN-FUNC: {{^}}vccz_workaround: ; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+:[0-9]+}}], 0x0 -; GCN: v_cmp_neq_f32_e64 [[MASK:s\[[0-9]+:[0-9]+\]]], 0 -; GCN: s_and_b64 vcc, exec, [[MASK]] +; GCN: v_cmp_neq_f32_e64 vcc, s{{[0-9]+}}, 0{{$}} ; GCN: s_waitcnt lgkmcnt(0) ; VCCZ-BUG: s_mov_b64 vcc, vcc ; NOVCCZ-BUG-NOT: s_mov_b64 vcc, vcc @@ -29,7 +28,6 @@ endif: ; GCN-FUNC: {{^}}vccz_noworkaround: ; GCN: v_cmp_neq_f32_e32 vcc, 0, v{{[0-9]+}} -; GCN: s_and_b64 vcc, exec, vcc ; GCN: s_cbranch_vccnz [[EXIT:[0-9A-Za-z_]+]] ; GCN: buffer_store_dword ; GCN: [[EXIT]]: diff --git a/test/CodeGen/AMDGPU/sopk-compares.ll b/test/CodeGen/AMDGPU/sopk-compares.ll new file mode 100644 index 000000000000..6b708d9af5db --- /dev/null +++ b/test/CodeGen/AMDGPU/sopk-compares.ll @@ -0,0 +1,649 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; Since this intrinsic is exposed as a constant after isel, use it to +; defeat the DAG's compare with constant canonicalizations. +declare i32 @llvm.amdgcn.groupstaticsize() #1 + +@lds = addrspace(3) global [512 x i32] undef, align 4 + +; GCN-LABEL: {{^}}br_scc_eq_i32_inline_imm: +; GCN: s_cmp_eq_u32 s{{[0-9]+}}, 4{{$}} +define void @br_scc_eq_i32_inline_imm(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 4 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_simm16_max: +; GCN: s_cmpk_eq_i32 s{{[0-9]+}}, 0x7fff{{$}} +define void @br_scc_eq_i32_simm16_max(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 32767 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_simm16_max_p1: +; GCN: s_cmpk_eq_u32 s{{[0-9]+}}, 0x8000{{$}} +define void @br_scc_eq_i32_simm16_max_p1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 32768 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ne_i32_simm16_max_p1: +; GCN: s_cmpk_lg_u32 s{{[0-9]+}}, 0x8000{{$}} +define void @br_scc_ne_i32_simm16_max_p1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ne i32 %cond, 32768 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_simm16_min: +; GCN: s_cmpk_eq_i32 s{{[0-9]+}}, 0x8000{{$}} +define void @br_scc_eq_i32_simm16_min(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, -32768 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_simm16_min_m1: +; GCN: s_cmp_eq_u32 s{{[0-9]+}}, 0xffff7fff{{$}} +define void @br_scc_eq_i32_simm16_min_m1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, -32769 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_uimm15_max: +; GCN: s_cmpk_eq_u32 s{{[0-9]+}}, 0xffff{{$}} +define void @br_scc_eq_i32_uimm15_max(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 65535 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_uimm16_max: +; GCN: s_cmpk_eq_u32 s{{[0-9]+}}, 0xffff{{$}} +define void @br_scc_eq_i32_uimm16_max(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 65535 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i32_uimm16_max_p1: +; GCN: s_cmp_eq_u32 s{{[0-9]+}}, 0x10000{{$}} +define void @br_scc_eq_i32_uimm16_max_p1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 65536 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + + +; GCN-LABEL: {{^}}br_scc_eq_i32: +; GCN: s_cmpk_eq_i32 s{{[0-9]+}}, 0x41{{$}} +define void @br_scc_eq_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i32 %cond, 65 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ne_i32: +; GCN: s_cmpk_lg_i32 s{{[0-9]+}}, 0x41{{$}} +define void @br_scc_ne_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ne i32 %cond, 65 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_sgt_i32: +; GCN: s_cmpk_gt_i32 s{{[0-9]+}}, 0x41{{$}} +define void @br_scc_sgt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp sgt i32 %cond, 65 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_sgt_i32_simm16_max: +; GCN: s_cmpk_gt_i32 s{{[0-9]+}}, 0x7fff{{$}} +define void @br_scc_sgt_i32_simm16_max(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp sgt i32 %cond, 32767 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_sgt_i32_simm16_max_p1: +; GCN: s_cmp_gt_i32 s{{[0-9]+}}, 0x8000{{$}} +define void @br_scc_sgt_i32_simm16_max_p1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp sgt i32 %cond, 32768 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_sge_i32: +; GCN: s_cmpk_ge_i32 s{{[0-9]+}}, 0x800{{$}} +define void @br_scc_sge_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp sge i32 %cond, %size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_slt_i32: +; GCN: s_cmpk_lt_i32 s{{[0-9]+}}, 0x41{{$}} +define void @br_scc_slt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp slt i32 %cond, 65 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_sle_i32: +; GCN: s_cmpk_le_i32 s{{[0-9]+}}, 0x800{{$}} +define void @br_scc_sle_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp sle i32 %cond, %size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ugt_i32: +; GCN: s_cmpk_gt_u32 s{{[0-9]+}}, 0x800{{$}} +define void @br_scc_ugt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ugt i32 %cond, %size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_uge_i32: +; GCN: s_cmpk_ge_u32 s{{[0-9]+}}, 0x800{{$}} +define void @br_scc_uge_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp uge i32 %cond, %size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ult_i32: +; GCN: s_cmpk_lt_u32 s{{[0-9]+}}, 0x41{{$}} +define void @br_scc_ult_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ult i32 %cond, 65 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ult_i32_min_simm16: +; GCN: s_cmp_lt_u32 s2, 0xffff8000 +define void @br_scc_ult_i32_min_simm16(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ult i32 %cond, -32768 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ult_i32_min_simm16_m1: +; GCN: s_cmp_lt_u32 s{{[0-9]+}}, 0xffff7fff{{$}} +define void @br_scc_ult_i32_min_simm16_m1(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ult i32 %cond, -32769 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ule_i32: +; GCN: s_cmpk_le_u32 s{{[0-9]+}}, 0x800{{$}} +define void @br_scc_ule_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ule i32 %cond, %size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_eq_i32: +; GCN: s_cmpk_eq_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_eq_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp eq i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_ne_i32: +; GCN: s_cmpk_lg_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_ne_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ne i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_sgt_i32: +; GCN: s_cmpk_lt_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_sgt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp sgt i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_sge_i32: +; GCN: s_cmpk_le_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_sge_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp sge i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_slt_i32: +; GCN: s_cmpk_gt_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_slt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp slt i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_sle_i32: +; GCN: s_cmpk_ge_i32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_sle_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp sle i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_ugt_i32: +; GCN: s_cmpk_lt_u32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_ugt_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ugt i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_uge_i32: +; GCN: s_cmpk_le_u32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_uge_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp uge i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_ult_i32: +; GCN: s_cmpk_gt_u32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_ult_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ult i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}commute_br_scc_ule_i32: +; GCN: s_cmpk_ge_u32 s{{[0-9]+}}, 0x800{{$}} +define void @commute_br_scc_ule_i32(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %cmp0 = icmp ule i32 %size, %cond + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ult_i32_non_u16: +; GCN: s_cmp_lt_u32 s2, 0xfffff7ff +define void @br_scc_ult_i32_non_u16(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %size = call i32 @llvm.amdgcn.groupstaticsize() + %not.size = xor i32 %size, -1 + %cmp0 = icmp ult i32 %cond, %not.size + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i64_inline_imm: +; VI: s_cmp_eq_u64 s{{\[[0-9]+:[0-9]+\]}}, 4 + +; SI: v_cmp_eq_u64_e64 +define void @br_scc_eq_i64_inline_imm(i64 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i64 %cond, 4 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_eq_i64_simm16: +; VI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x4d2 +; VI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0 +; VI: s_cmp_eq_u64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} + +; SI: v_cmp_eq_u64_e32 +define void @br_scc_eq_i64_simm16(i64 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp eq i64 %cond, 1234 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ne_i64_inline_imm: +; VI: s_cmp_lg_u64 s{{\[[0-9]+:[0-9]+\]}}, 4 + +; SI: v_cmp_ne_u64_e64 +define void @br_scc_ne_i64_inline_imm(i64 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ne i64 %cond, 4 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}br_scc_ne_i64_simm16: +; VI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x4d2 +; VI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0 +; VI: s_cmp_lg_u64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} + +; SI: v_cmp_ne_u64_e32 +define void @br_scc_ne_i64_simm16(i64 %cond, i32 addrspace(1)* %out) #0 { +entry: + %cmp0 = icmp ne i64 %cond, 1234 + br i1 %cmp0, label %endif, label %if + +if: + call void asm sideeffect "", ""() + br label %endif + +endif: + store volatile i32 1, i32 addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/spill-alloc-sgpr-init-bug.ll b/test/CodeGen/AMDGPU/spill-alloc-sgpr-init-bug.ll index cc4b6bcbfb51..ff9429843b22 100644 --- a/test/CodeGen/AMDGPU/spill-alloc-sgpr-init-bug.ll +++ b/test/CodeGen/AMDGPU/spill-alloc-sgpr-init-bug.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=TONGA %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck --check-prefix=TONGA %s ; On Tonga and Iceland, limited SGPR availability means care must be taken to ; allocate scratch registers correctly. Check that this test compiles without diff --git a/test/CodeGen/AMDGPU/spill-m0.ll b/test/CodeGen/AMDGPU/spill-m0.ll new file mode 100644 index 000000000000..8c16b9d1649c --- /dev/null +++ b/test/CodeGen/AMDGPU/spill-m0.ll @@ -0,0 +1,214 @@ +; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -march=amdgcn -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s +; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s +; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -march=amdgcn -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s +; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s +; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=1 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOSMEM -check-prefix=GCN %s + +; XXX - Why does it like to use vcc? + +; GCN-LABEL: {{^}}spill_m0: +; TOSMEM: s_mov_b32 s[[LO:[0-9]+]], SCRATCH_RSRC_DWORD0 +; TOSMEM: s_mov_b32 s[[HI:[0-9]+]], 0xe80000 + +; GCN-DAG: s_cmp_lg_u32 + +; TOVGPR-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 +; TOVGPR: v_writelane_b32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]], 0 + +; TOVMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 +; TOVMEM-DAG: v_mov_b32_e32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]] +; TOVMEM: buffer_store_dword [[SPILL_VREG]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} ; 4-byte Folded Spill +; TOVMEM: s_waitcnt vmcnt(0) + +; TOSMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 +; TOSMEM: s_mov_b32 m0, s3{{$}} +; TOSMEM-NOT: [[M0_COPY]] +; TOSMEM: s_buffer_store_dword [[M0_COPY]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Spill +; TOSMEM: s_waitcnt lgkmcnt(0) + +; GCN: s_cbranch_scc1 [[ENDIF:BB[0-9]+_[0-9]+]] + +; GCN: [[ENDIF]]: +; TOVGPR: v_readlane_b32 [[M0_RESTORE:s[0-9]+]], [[SPILL_VREG]], 0 +; TOVGPR: s_mov_b32 m0, [[M0_RESTORE]] + +; TOVMEM: buffer_load_dword [[RELOAD_VREG:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} ; 4-byte Folded Reload +; TOVMEM: s_waitcnt vmcnt(0) +; TOVMEM: v_readfirstlane_b32 [[M0_RESTORE:s[0-9]+]], [[RELOAD_VREG]] +; TOVMEM: s_mov_b32 m0, [[M0_RESTORE]] + +; TOSMEM: s_mov_b32 m0, s3{{$}} +; TOSMEM: s_buffer_load_dword [[M0_RESTORE:s[0-9]+]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Reload +; TOSMEM-NOT: [[M0_RESTORE]] +; TOSMEM: s_mov_b32 m0, [[M0_RESTORE]] + +; GCN: s_add_i32 s{{[0-9]+}}, m0, 1 +define void @spill_m0(i32 %cond, i32 addrspace(1)* %out) #0 { +entry: + %m0 = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0 + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %if, label %endif + +if: + call void asm sideeffect "v_nop", ""() #0 + br label %endif + +endif: + %foo = call i32 asm sideeffect "s_add_i32 $0, $1, 1", "=s,{M0}"(i32 %m0) #0 + store i32 %foo, i32 addrspace(1)* %out + ret void +} + +@lds = internal addrspace(3) global [64 x float] undef + +; m0 is killed, so it isn't necessary during the entry block spill to preserve it +; GCN-LABEL: {{^}}spill_kill_m0_lds: +; GCN: s_mov_b32 m0, s6 +; GCN: v_interp_mov_f32 + +; TOSMEM-NOT: s_m0 +; TOSMEM: s_mov_b32 m0, s7 +; TOSMEM-NEXT: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill +; TOSMEM-NOT: m0 + +; TOSMEM-NOT: m0 +; TOSMEM: s_add_u32 m0, s7, 0x100 +; TOSMEM: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill +; TOSMEM-NOT: m0 + +; TOSMEM: s_mov_b64 exec, +; TOSMEM: s_cbranch_execz +; TOSMEM: s_branch + +; TOSMEM: BB{{[0-9]+_[0-9]+}}: +; TOSMEM-NEXT: s_add_u32 m0, s7, 0x100 +; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload + + +; GCN-NOT: v_readlane_b32 m0 +; GCN-NOT: s_buffer_store_dword m0 +; GCN-NOT: s_buffer_load_dword m0 +define amdgpu_ps void @spill_kill_m0_lds(<16 x i8> addrspace(2)* inreg %arg, <16 x i8> addrspace(2)* inreg %arg1, <32 x i8> addrspace(2)* inreg %arg2, i32 inreg %arg3) #0 { +main_body: + %tmp = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %arg3) + %cmp = fcmp ueq float 0.000000e+00, %tmp + br i1 %cmp, label %if, label %else + +if: ; preds = %main_body + %lds_ptr = getelementptr [64 x float], [64 x float] addrspace(3)* @lds, i32 0, i32 0 + %lds_data = load float, float addrspace(3)* %lds_ptr + br label %endif + +else: ; preds = %main_body + %interp = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %arg3) + br label %endif + +endif: ; preds = %else, %if + %export = phi float [ %lds_data, %if ], [ %interp, %else ] + %tmp4 = call i32 @llvm.SI.packf16(float %export, float %export) + %tmp5 = bitcast i32 %tmp4 to float + call void @llvm.SI.export(i32 15, i32 1, i32 1, i32 0, i32 1, float %tmp5, float %tmp5, float %tmp5, float %tmp5) + ret void +} + +; Force save and restore of m0 during SMEM spill +; GCN-LABEL: {{^}}m0_unavailable_spill: + +; GCN: ; def m0, 1 + +; GCN: s_mov_b32 m0, s2 +; GCN: v_interp_mov_f32 + +; GCN: ; clobber m0 + +; TOSMEM: s_mov_b32 vcc_hi, m0 +; TOSMEM: s_mov_b32 m0, s3 +; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill +; TOSMEM: s_mov_b32 m0, vcc_hi + +; TOSMEM: s_mov_b64 exec, +; TOSMEM: s_cbranch_execz +; TOSMEM: s_branch + +; TOSMEM: BB{{[0-9]+_[0-9]+}}: +; TOSMEM-NEXT: s_mov_b32 m0, s3 +; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload + +; GCN-NOT: v_readlane_b32 m0 +; GCN-NOT: s_buffer_store_dword m0 +; GCN-NOT: s_buffer_load_dword m0 +define void @m0_unavailable_spill(i32 %arg3) #0 { +main_body: + %m0 = call i32 asm sideeffect "; def $0, 1", "={M0}"() #0 + %tmp = call float @llvm.SI.fs.constant(i32 0, i32 0, i32 %arg3) + call void asm sideeffect "; clobber $0", "~{M0}"() #0 + %cmp = fcmp ueq float 0.000000e+00, %tmp + br i1 %cmp, label %if, label %else + +if: ; preds = %main_body + store volatile i32 8, i32 addrspace(1)* undef + br label %endif + +else: ; preds = %main_body + store volatile i32 11, i32 addrspace(1)* undef + br label %endif + +endif: + ret void +} + +; GCN-LABEL: {{^}}restore_m0_lds: +; TOSMEM: s_load_dwordx2 [[REG:s\[[0-9]+:[0-9]+\]]] +; TOSMEM: s_cmp_eq_u32 +; TOSMEM-NOT: m0 +; TOSMEM: s_mov_b32 m0, s3 +; TOSMEM: s_buffer_store_dwordx2 [[REG]], s[88:91], m0 ; 8-byte Folded Spill +; TOSMEM-NOT: m0 +; TOSMEM: s_add_u32 m0, s3, 0x200 +; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s[88:91], m0 ; 4-byte Folded Spill +; TOSMEM-NOT: m0 +; TOSMEM: s_cbranch_scc1 + +; TOSMEM: s_mov_b32 m0, -1 + +; TOSMEM: s_mov_b32 vcc_hi, m0 +; TOSMEM: s_mov_b32 m0, s3 +; TOSMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[88:91], m0 ; 8-byte Folded Reload +; TOSMEM: s_mov_b32 m0, vcc_hi +; TOSMEM: s_waitcnt lgkmcnt(0) + +; TOSMEM: ds_write_b64 + +; TOSMEM-NOT: m0 +; TOSMEM: s_add_u32 m0, s3, 0x200 +; TOSMEM: s_buffer_load_dword s0, s[88:91], m0 ; 4-byte Folded Reload +; TOSMEM-NOT: m0 +; TOSMEM: s_waitcnt lgkmcnt(0) +; TOSMEM-NOT: m0 +; TOSMEM: s_mov_b32 m0, s0 +; TOSMEM: ; use m0 + +; TOSMEM: s_dcache_wb +; TOSMEM: s_endpgm +define void @restore_m0_lds(i32 %arg) { + %m0 = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0 + %sval = load volatile i64, i64 addrspace(2)* undef + %cmp = icmp eq i32 %arg, 0 + br i1 %cmp, label %ret, label %bb + +bb: + store volatile i64 %sval, i64 addrspace(3)* undef + call void asm sideeffect "; use $0", "{M0}"(i32 %m0) #0 + br label %ret + +ret: + ret void +} + +declare float @llvm.SI.fs.constant(i32, i32, i32) readnone + +declare i32 @llvm.SI.packf16(float, float) readnone + +declare void @llvm.SI.export(i32, i32, i32, i32, i32, float, float, float, float) + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/spill-wide-sgpr.ll b/test/CodeGen/AMDGPU/spill-wide-sgpr.ll new file mode 100644 index 000000000000..cab45be8da50 --- /dev/null +++ b/test/CodeGen/AMDGPU/spill-wide-sgpr.ll @@ -0,0 +1,176 @@ +; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VGPR %s +; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=1 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=SMEM %s +; RUN: llc -O0 -march=amdgcn -mcpu=fiji -amdgpu-spill-sgpr-to-smem=0 -amdgpu-spill-sgpr-to-vgpr=0 -verify-machineinstrs < %s | FileCheck -check-prefix=ALL -check-prefix=VMEM %s + +; ALL-LABEL: {{^}}spill_sgpr_x2: +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[8:11], m0 ; 8-byte Folded Spill +; SMEM: s_cbranch_scc1 + +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[8:11], m0 ; 8-byte Folded Reload + +; SMEM: s_dcache_wb +; SMEM: s_endpgm + +; FIXME: Should only need 4 bytes +; SMEM: ScratchSize: 12 + + +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1 +; VGPR: s_cbranch_scc1 + +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 0 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 1 + +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: s_cbranch_scc1 + +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +define amdgpu_kernel void @spill_sgpr_x2(i32 addrspace(1)* %out, i32 %in) #0 { + %wide.sgpr = call <2 x i32> asm sideeffect "; def $0", "=s" () #0 + %cmp = icmp eq i32 %in, 0 + br i1 %cmp, label %bb0, label %ret + +bb0: + call void asm sideeffect "; use $0", "s"(<2 x i32> %wide.sgpr) #0 + br label %ret + +ret: + ret void +} + +; ALL-LABEL: {{^}}spill_sgpr_x4: +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[12:15], m0 ; 16-byte Folded Spill +; SMEM: s_cbranch_scc1 + +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[12:15], m0 ; 16-byte Folded Reload +; SMEM: s_dcache_wb +; SMEM: s_endpgm + +; FIXME: Should only need 4 bytes +; SMEM: ScratchSize: 20 + +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 2 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 3 +; VGPR: s_cbranch_scc1 + +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 0 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 1 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 2 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 3 + + +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: s_cbranch_scc1 + +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +define amdgpu_kernel void @spill_sgpr_x4(i32 addrspace(1)* %out, i32 %in) #0 { + %wide.sgpr = call <4 x i32> asm sideeffect "; def $0", "=s" () #0 + %cmp = icmp eq i32 %in, 0 + br i1 %cmp, label %bb0, label %ret + +bb0: + call void asm sideeffect "; use $0", "s"(<4 x i32> %wide.sgpr) #0 + br label %ret + +ret: + ret void +} + +; ALL-LABEL: {{^}}spill_sgpr_x8: + +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[16:19], m0 ; 16-byte Folded Spill +; SMEM: s_add_u32 m0, s3, 16 +; SMEM: s_buffer_store_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[16:19], m0 ; 16-byte Folded Spill +; SMEM: s_cbranch_scc1 + +; SMEM: s_mov_b32 m0, s3{{$}} +; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[16:19], m0 ; 16-byte Folded Reload +; SMEM: s_add_u32 m0, s3, 16 +; SMEM: s_buffer_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s[16:19], m0 ; 16-byte Folded Reload + +; SMEM: s_dcache_wb +; SMEM: s_endpgm + +; SMEM: ScratchSize: 36 + +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 0 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 1 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 2 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 3 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 4 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 5 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 6 +; VGPR: v_writelane_b32 v{{[0-9]+}}, s{{[0-9]+}}, 7 +; VGPR: s_cbranch_scc1 + +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 0 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 1 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 2 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 3 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 4 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 5 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 6 +; VGPR: v_readlane_b32 s{{[0-9]+}}, v{{[0-9]+}}, 7 + +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: buffer_store_dword +; VMEM: s_cbranch_scc1 + +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +; VMEM: buffer_load_dword +define amdgpu_kernel void @spill_sgpr_x8(i32 addrspace(1)* %out, i32 %in) #0 { + %wide.sgpr = call <8 x i32> asm sideeffect "; def $0", "=s" () #0 + %cmp = icmp eq i32 %in, 0 + br i1 %cmp, label %bb0, label %ret + +bb0: + call void asm sideeffect "; use $0", "s"(<8 x i32> %wide.sgpr) #0 + br label %ret + +ret: + ret void +} + +; FIXME: x16 inlineasm seems broken +; define amdgpu_kernel void @spill_sgpr_x16(i32 addrspace(1)* %out, i32 %in) #0 { +; %wide.sgpr = call <16 x i32> asm sideeffect "; def $0", "=s" () #0 +; %cmp = icmp eq i32 %in, 0 +; br i1 %cmp, label %bb0, label %ret + +; bb0: +; call void asm sideeffect "; use $0", "s"(<16 x i32> %wide.sgpr) #0 +; br label %ret + +; ret: +; ret void +; } + +attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/split-smrd.ll b/test/CodeGen/AMDGPU/split-smrd.ll index 237a62c1360a..d07da1030936 100644 --- a/test/CodeGen/AMDGPU/split-smrd.ll +++ b/test/CodeGen/AMDGPU/split-smrd.ll @@ -42,5 +42,6 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !3} !2 = !{!1, !1, i64 0} +!3 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll b/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll index 484150bc25fd..37ec2b012896 100644 --- a/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll +++ b/test/CodeGen/AMDGPU/split-vector-memoperand-offsets.ll @@ -1,7 +1,6 @@ -; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs -mattr=-promote-alloca < %s | FileCheck -check-prefix=GCN %s -; XFAIL: * +; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs -mattr=-promote-alloca,-load-store-opt < %s | FileCheck -check-prefix=GCN %s -@sPrivateStorage = external addrspace(3) global [256 x [8 x <4 x i64>]] +@sPrivateStorage = internal addrspace(3) global [256 x [8 x <4 x i64>]] undef ; GCN-LABEL: {{^}}ds_reorder_vector_split: @@ -16,20 +15,19 @@ ; GCN-DAG: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:24 ; GCN-DAG: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:16 ; GCN-DAG: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]}} offset:8 +; Appears to be dead store of vector component. +; GCN-DAG: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]$}} -; GCN: s_waitcnt lgkmcnt -; GCN-DAG ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:8 +; GCN-DAG: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:8 ; GCN-DAG: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:16 ; GCN-DAG: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:24 -; Appears to be dead store of vector component. -; GCN: ds_write_b64 v{{[0-9]+}}, {{v\[[0-9]+:[0-9]+\]$}} +; GCN-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}} +; GCN-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 +; GCN-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 +; GCN-DAG: buffer_store_dwordx2 {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 -; GCN: buffer_store_dwordx2 -; GCN: buffer_store_dwordx2 -; GCN: buffer_store_dwordx2 -; GCN: buffer_store_dwordx2 ; GCN: s_endpgm define void @ds_reorder_vector_split(<4 x i64> addrspace(1)* nocapture readonly %srcValues, i32 addrspace(1)* nocapture readonly %offsets, <4 x i64> addrspace(1)* nocapture %destBuffer, i32 %alignmentOffset) #0 { entry: diff --git a/test/CodeGen/AMDGPU/sra.ll b/test/CodeGen/AMDGPU/sra.ll index dddfbfd3ed10..710547426e3c 100644 --- a/test/CodeGen/AMDGPU/sra.ll +++ b/test/CodeGen/AMDGPU/sra.ll @@ -46,6 +46,36 @@ define void @ashr_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %i ret void } +; FUNC-LABEL: {{^}}ashr_v2i16: +; FIXME: The ashr operation is uniform, but because its operands come from a +; global load we end up with the vector instructions rather than scalar. +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +define void @ashr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) { + %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i16 1 + %a = load <2 x i16>, <2 x i16> addrspace(1)* %in + %b = load <2 x i16>, <2 x i16> addrspace(1)* %b_ptr + %result = ashr <2 x i16> %a, %b + store <2 x i16> %result, <2 x i16> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}ashr_v4i16: +; FIXME: The ashr operation is uniform, but because its operands come from a +; global load we end up with the vector instructions rather than scalar. +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_ashrrev_i32_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +define void @ashr_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) { + %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i16 1 + %a = load <4 x i16>, <4 x i16> addrspace(1)* %in + %b = load <4 x i16>, <4 x i16> addrspace(1)* %b_ptr + %result = ashr <4 x i16> %a, %b + store <4 x i16> %result, <4 x i16> addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}s_ashr_i64: ; GCN: s_ashr_i64 s[{{[0-9]}}:{{[0-9]}}], s[{{[0-9]}}:{{[0-9]}}], 8 diff --git a/test/CodeGen/AMDGPU/store.ll b/test/CodeGen/AMDGPU/store-global.ll index 68c659fdd2ff..0e68cf450578 100644 --- a/test/CodeGen/AMDGPU/store.ll +++ b/test/CodeGen/AMDGPU/store-global.ll @@ -1,14 +1,16 @@ -; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s ; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s -;===------------------------------------------------------------------------===; -; Global Address Space -;===------------------------------------------------------------------------===; ; FUNC-LABEL: {{^}}store_i1: ; EG: MEM_RAT MSKOR -; SI: buffer_store_byte +; EG-NOT: MEM_RAT MSKOR + +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR + +; GCN: buffer_store_byte define void @store_i1(i1 addrspace(1)* %out) { entry: store i1 true, i1 addrspace(1)* %out @@ -18,6 +20,7 @@ entry: ; i8 store ; FUNC-LABEL: {{^}}store_i8: ; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X +; EG-NOT: MEM_RAT MSKOR ; IG 0: Get the byte index and truncate the value ; EG: AND_INT * T{{[0-9]}}.[[BI_CHAN:[XYZW]]], KC0[2].Y, literal.x @@ -37,7 +40,7 @@ entry: ; EG: MOV T[[RW_GPR]].Y, 0.0 ; EG: MOV * T[[RW_GPR]].Z, 0.0 -; SI: buffer_store_byte +; GCN: buffer_store_byte define void @store_i8(i8 addrspace(1)* %out, i8 %in) { entry: @@ -48,6 +51,7 @@ entry: ; i16 store ; FUNC-LABEL: {{^}}store_i16: ; EG: MEM_RAT MSKOR T[[RW_GPR:[0-9]]].XW, T{{[0-9]}}.X +; EG-NOT: MEM_RAT MSKOR ; IG 0: Get the byte index and truncate the value @@ -70,7 +74,7 @@ entry: ; EG: MOV T[[RW_GPR]].Y, 0.0 ; EG: MOV * T[[RW_GPR]].Z, 0.0 -; SI: buffer_store_short +; GCN: buffer_store_short define void @store_i16(i16 addrspace(1)* %out, i16 %in) { entry: store i16 %in, i16 addrspace(1)* %out @@ -78,9 +82,12 @@ entry: } ; FUNC-LABEL: {{^}}store_i24: -; SI: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 -; SI-DAG: buffer_store_byte -; SI-DAG: buffer_store_short +; GCN: s_lshr_b32 s{{[0-9]+}}, s{{[0-9]+}}, 16 +; GCN-DAG: buffer_store_byte +; GCN-DAG: buffer_store_short + +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR define void @store_i24(i24 addrspace(1)* %out, i24 %in) { entry: store i24 %in, i24 addrspace(1)* %out @@ -88,9 +95,15 @@ entry: } ; FUNC-LABEL: {{^}}store_i25: -; SI: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 0x1ffffff{{$}} -; SI: v_mov_b32_e32 [[VAND:v[0-9]+]], [[AND]] -; SI: buffer_store_dword [[VAND]] +; GCN: s_and_b32 [[AND:s[0-9]+]], s{{[0-9]+}}, 0x1ffffff{{$}} +; GCN: v_mov_b32_e32 [[VAND:v[0-9]+]], [[AND]] +; GCN: buffer_store_dword [[VAND]] + +; EG: MEM_RAT_CACHELESS STORE_RAW +; EG-NOT: MEM_RAT + +; CM: MEM_RAT_CACHELESS STORE_DWORD +; CM-NOT: MEM_RAT define void @store_i25(i25 addrspace(1)* %out, i25 %in) { entry: store i25 %in, i25 addrspace(1)* %out @@ -98,10 +111,14 @@ entry: } ; FUNC-LABEL: {{^}}store_v2i8: +; v2i8 is naturally 2B aligned ; EG: MEM_RAT MSKOR ; EG-NOT: MEM_RAT MSKOR -; SI: buffer_store_short +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR + +; GCN: buffer_store_short define void @store_v2i8(<2 x i8> addrspace(1)* %out, <2 x i32> %in) { entry: %0 = trunc <2 x i32> %in to <2 x i8> @@ -109,13 +126,30 @@ entry: ret void } +; FUNC-LABEL: {{^}}store_v2i8_unaligned: +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG-NOT: MEM_RAT MSKOR + +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR + +; SI: buffer_store_byte +define void @store_v2i8_unaligned(<2 x i8> addrspace(1)* %out, <2 x i32> %in) { +entry: + %0 = trunc <2 x i32> %in to <2 x i8> + store <2 x i8> %0, <2 x i8> addrspace(1)* %out, align 1 + ret void +} + ; FUNC-LABEL: {{^}}store_v2i16: ; EG: MEM_RAT_CACHELESS STORE_RAW ; CM: MEM_RAT_CACHELESS STORE_DWORD -; SI: buffer_store_dword +; GCN: buffer_store_dword define void @store_v2i16(<2 x i16> addrspace(1)* %out, <2 x i32> %in) { entry: %0 = trunc <2 x i32> %in to <2 x i16> @@ -123,12 +157,32 @@ entry: ret void } +; FUNC-LABEL: {{^}}store_v2i16_unaligned: +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG-NOT: MEM_RAT MSKOR +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW + +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD + +; SI: buffer_store_short +; SI: buffer_store_short +define void @store_v2i16_unaligned(<2 x i16> addrspace(1)* %out, <2 x i32> %in) { +entry: + %0 = trunc <2 x i32> %in to <2 x i16> + store <2 x i16> %0, <2 x i16> addrspace(1)* %out, align 2 + ret void +} + ; FUNC-LABEL: {{^}}store_v4i8: ; EG: MEM_RAT_CACHELESS STORE_RAW ; CM: MEM_RAT_CACHELESS STORE_DWORD -; SI: buffer_store_dword +; GCN: buffer_store_dword define void @store_v4i8(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { entry: %0 = trunc <4 x i32> %in to <4 x i8> @@ -136,13 +190,61 @@ entry: ret void } +; FUNC-LABEL: {{^}}store_v4i8_unaligned: +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG-NOT: MEM_RAT MSKOR +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW + +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD + +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI: buffer_store_byte +; SI-NOT: buffer_store_dword +define void @store_v4i8_unaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { +entry: + %0 = trunc <4 x i32> %in to <4 x i8> + store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 1 + ret void +} + +; FUNC-LABEL: {{^}}store_v4i8_halfaligned: +; EG: MEM_RAT MSKOR +; EG: MEM_RAT MSKOR +; EG-NOT: MEM_RAT MSKOR +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW + +; CM: MEM_RAT MSKOR +; CM: MEM_RAT MSKOR +; CM-NOT: MEM_RAT MSKOR +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD + +; SI: buffer_store_short +; SI: buffer_store_short +; SI-NOT: buffer_store_dword +define void @store_v4i8_halfaligned(<4 x i8> addrspace(1)* %out, <4 x i32> %in) { +entry: + %0 = trunc <4 x i32> %in to <4 x i8> + store <4 x i8> %0, <4 x i8> addrspace(1)* %out, align 2 + ret void +} + ; floating-point store ; FUNC-LABEL: {{^}}store_f32: ; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.X, T[0-9]+\.X}}, 1 ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+\.X, T[0-9]+\.X}} -; SI: buffer_store_dword +; GCN: buffer_store_dword define void @store_f32(float addrspace(1)* %out, float %in) { store float %in, float addrspace(1)* %out @@ -150,9 +252,11 @@ define void @store_f32(float addrspace(1)* %out, float %in) { } ; FUNC-LABEL: {{^}}store_v4i16: -; MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XYZW +; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+}}.XY + +; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}} -; SI: buffer_store_dwordx2 +; GCN: buffer_store_dwordx2 define void @store_v4i16(<4 x i16> addrspace(1)* %out, <4 x i32> %in) { entry: %0 = trunc <4 x i32> %in to <4 x i16> @@ -166,7 +270,7 @@ entry: ; CM: MEM_RAT_CACHELESS STORE_DWORD -; SI: buffer_store_dwordx2 +; GCN: buffer_store_dwordx2 define void @store_v2f32(<2 x float> addrspace(1)* %out, float %a, float %b) { entry: @@ -176,179 +280,95 @@ entry: ret void } +; FUNC-LABEL: {{^}}store_v3i32: +; GCN-DAG: buffer_store_dwordx2 +; GCN-DAG: buffer_store_dword v + +; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.[XYZW]}}, {{T[0-9]+\.[XYZW]}}, +; EG-DAG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XY}}, {{T[0-9]+\.[XYZW]}}, +define void @store_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a) nounwind { + store <3 x i32> %a, <3 x i32> addrspace(1)* %out, align 16 + ret void +} + ; FUNC-LABEL: {{^}}store_v4i32: -; EG: MEM_RAT_CACHELESS STORE_RAW +; EG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XYZW}} ; EG-NOT: MEM_RAT_CACHELESS STORE_RAW ; CM: MEM_RAT_CACHELESS STORE_DWORD ; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD -; SI: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 define void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> %in) { entry: store <4 x i32> %in, <4 x i32> addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}store_i64_i8: -; EG: MEM_RAT MSKOR -; SI: buffer_store_byte -define void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) { -entry: - %0 = trunc i64 %in to i8 - store i8 %0, i8 addrspace(1)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_i64_i16: -; EG: MEM_RAT MSKOR -; SI: buffer_store_short -define void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) { -entry: - %0 = trunc i64 %in to i16 - store i16 %0, i16 addrspace(1)* %out - ret void -} - -;===------------------------------------------------------------------------===; -; Local Address Space -;===------------------------------------------------------------------------===; - -; FUNC-LABEL: {{^}}store_local_i1: -; EG: LDS_BYTE_WRITE -; SI: ds_write_b8 -define void @store_local_i1(i1 addrspace(3)* %out) { -entry: - store i1 true, i1 addrspace(3)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_local_i8: -; EG: LDS_BYTE_WRITE - -; SI: ds_write_b8 -define void @store_local_i8(i8 addrspace(3)* %out, i8 %in) { - store i8 %in, i8 addrspace(3)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_local_i16: -; EG: LDS_SHORT_WRITE - -; SI: ds_write_b16 -define void @store_local_i16(i16 addrspace(3)* %out, i16 %in) { - store i16 %in, i16 addrspace(3)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_local_v2i16: -; EG: LDS_WRITE - -; CM: LDS_WRITE - -; SI: ds_write_b32 -define void @store_local_v2i16(<2 x i16> addrspace(3)* %out, <2 x i16> %in) { -entry: - store <2 x i16> %in, <2 x i16> addrspace(3)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_local_v4i8: -; EG: LDS_WRITE - -; CM: LDS_WRITE - -; SI: ds_write_b32 -define void @store_local_v4i8(<4 x i8> addrspace(3)* %out, <4 x i8> %in) { -entry: - store <4 x i8> %in, <4 x i8> addrspace(3)* %out - ret void -} - -; FUNC-LABEL: {{^}}store_local_v2i32: -; EG: LDS_WRITE -; EG: LDS_WRITE +; FUNC-LABEL: {{^}}store_v4i32_unaligned: +; EG: MEM_RAT_CACHELESS STORE_RAW {{T[0-9]+\.XYZW}} +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW -; CM: LDS_WRITE -; CM: LDS_WRITE +; CM: MEM_RAT_CACHELESS STORE_DWORD +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD -; SI: ds_write_b64 -define void @store_local_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> %in) { +; SI: buffer_store_dwordx4 +define void @store_v4i32_unaligned(<4 x i32> addrspace(1)* %out, <4 x i32> %in) { entry: - store <2 x i32> %in, <2 x i32> addrspace(3)* %out + store <4 x i32> %in, <4 x i32> addrspace(1)* %out, align 4 ret void } -; FUNC-LABEL: {{^}}store_local_v4i32: -; EG: LDS_WRITE -; EG: LDS_WRITE -; EG: LDS_WRITE -; EG: LDS_WRITE +; v4f32 store +; FUNC-LABEL: {{^}}store_v4f32: +; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XYZW, T[0-9]+\.X}}, 1 +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW -; CM: LDS_WRITE -; CM: LDS_WRITE -; CM: LDS_WRITE -; CM: LDS_WRITE +; CM: MEM_RAT_CACHELESS STORE_DWORD +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD -; SI: ds_write2_b64 -define void @store_local_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %in) { -entry: - store <4 x i32> %in, <4 x i32> addrspace(3)* %out +; GCN: buffer_store_dwordx4 +define void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) { + %1 = load <4 x float>, <4 x float> addrspace(1) * %in + store <4 x float> %1, <4 x float> addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}store_local_v4i32_align4: -; EG: LDS_WRITE -; EG: LDS_WRITE -; EG: LDS_WRITE -; EG: LDS_WRITE +; FUNC-LABEL: {{^}}store_i64_i8: +; EG: MEM_RAT MSKOR -; CM: LDS_WRITE -; CM: LDS_WRITE -; CM: LDS_WRITE -; CM: LDS_WRITE +; CM: MEM_RAT MSKOR -; SI: ds_write2_b32 -; SI: ds_write2_b32 -define void @store_local_v4i32_align4(<4 x i32> addrspace(3)* %out, <4 x i32> %in) { -entry: - store <4 x i32> %in, <4 x i32> addrspace(3)* %out, align 4 - ret void -} - -; FUNC-LABEL: {{^}}store_local_i64_i8: -; EG: LDS_BYTE_WRITE -; SI: ds_write_b8 -define void @store_local_i64_i8(i8 addrspace(3)* %out, i64 %in) { +; GCN: buffer_store_byte +define void @store_i64_i8(i8 addrspace(1)* %out, i64 %in) { entry: %0 = trunc i64 %in to i8 - store i8 %0, i8 addrspace(3)* %out + store i8 %0, i8 addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}store_local_i64_i16: -; EG: LDS_SHORT_WRITE -; SI: ds_write_b16 -define void @store_local_i64_i16(i16 addrspace(3)* %out, i64 %in) { +; FUNC-LABEL: {{^}}store_i64_i16: +; EG: MEM_RAT MSKOR +; GCN: buffer_store_short +define void @store_i64_i16(i16 addrspace(1)* %out, i64 %in) { entry: %0 = trunc i64 %in to i16 - store i16 %0, i16 addrspace(3)* %out + store i16 %0, i16 addrspace(1)* %out ret void } ; The stores in this function are combined by the optimizer to create a -; 64-bit store with 32-bit alignment. This is legal for SI and the legalizer +; 64-bit store with 32-bit alignment. This is legal and the legalizer ; should not try to split the 64-bit store back into 2 32-bit stores. -; -; Evergreen / Northern Islands don't support 64-bit stores yet, so there should -; be two 32-bit stores. ; FUNC-LABEL: {{^}}vecload2: -; EG: MEM_RAT_CACHELESS STORE_RAW +; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XY, T[0-9]+\.X}}, 1 +; EG-NOT: MEM_RAT_CACHELESS STORE_RAW ; CM: MEM_RAT_CACHELESS STORE_DWORD +; CM-NOT: MEM_RAT_CACHELESS STORE_DWORD -; SI: buffer_store_dwordx2 +; GCN: buffer_store_dwordx2 define void @vecload2(i32 addrspace(1)* nocapture %out, i32 addrspace(2)* nocapture %mem) #0 { entry: %0 = load i32, i32 addrspace(2)* %mem, align 4 @@ -367,7 +387,7 @@ entry: ; CM: MEM_RAT_CACHELESS STORE_DWORD T{{[0-9]+}}, T{{[0-9]+}}.X -; SI: buffer_store_dwordx4 +; GCN: buffer_store_dwordx4 define void @i128-const-store(i32 addrspace(1)* %out) { entry: store i32 1, i32 addrspace(1)* %out, align 4 diff --git a/test/CodeGen/AMDGPU/store-local.ll b/test/CodeGen/AMDGPU/store-local.ll new file mode 100644 index 000000000000..03fd30ca9a25 --- /dev/null +++ b/test/CodeGen/AMDGPU/store-local.ll @@ -0,0 +1,175 @@ +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=cayman < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s + +; FUNC-LABEL: {{^}}store_local_i1: +; EG: LDS_BYTE_WRITE + +; CM: LDS_BYTE_WRITE + +; GCN: ds_write_b8 +define void @store_local_i1(i1 addrspace(3)* %out) { +entry: + store i1 true, i1 addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_i8: +; EG: LDS_BYTE_WRITE + +; CM: LDS_BYTE_WRITE + +; GCN: ds_write_b8 +define void @store_local_i8(i8 addrspace(3)* %out, i8 %in) { + store i8 %in, i8 addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_i16: +; EG: LDS_SHORT_WRITE + +; CM: LDS_SHORT_WRITE + +; GCN: ds_write_b16 +define void @store_local_i16(i16 addrspace(3)* %out, i16 %in) { + store i16 %in, i16 addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_v2i16: +; EG: LDS_WRITE + +; CM: LDS_WRITE + +; GCN: ds_write_b32 +define void @store_local_v2i16(<2 x i16> addrspace(3)* %out, <2 x i16> %in) { +entry: + store <2 x i16> %in, <2 x i16> addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_v4i8: +; EG: LDS_WRITE + +; CM: LDS_WRITE + +; GCN: ds_write_b32 +define void @store_local_v4i8(<4 x i8> addrspace(3)* %out, <4 x i8> %in) { +entry: + store <4 x i8> %in, <4 x i8> addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_v4i8_unaligned: +; EG: LDS_BYTE_WRITE +; EG: LDS_BYTE_WRITE +; EG: LDS_BYTE_WRITE +; EG: LDS_BYTE_WRITE +; EG-NOT: LDS_WRITE + +; CM: LDS_BYTE_WRITE +; CM: LDS_BYTE_WRITE +; CM: LDS_BYTE_WRITE +; CM: LDS_BYTE_WRITE +; CM-NOT: LDS_WRITE + +; GCN: ds_write_b8 +; GCN: ds_write_b8 +; GCN: ds_write_b8 +; GCN: ds_write_b8 +define void @store_local_v4i8_unaligned(<4 x i8> addrspace(3)* %out, <4 x i8> %in) { +entry: + store <4 x i8> %in, <4 x i8> addrspace(3)* %out, align 1 + ret void +} + +; FUNC-LABEL: {{^}}store_local_v4i8_halfaligned: +; EG: LDS_SHORT_WRITE +; EG: LDS_SHORT_WRITE +; EG-NOT: LDS_WRITE + +; CM: LDS_SHORT_WRITE +; CM: LDS_SHORT_WRITE +; CM-NOT: LDS_WRITE + +; GCN: ds_write_b16 +; GCN: ds_write_b16 +define void @store_local_v4i8_halfaligned(<4 x i8> addrspace(3)* %out, <4 x i8> %in) { +entry: + store <4 x i8> %in, <4 x i8> addrspace(3)* %out, align 2 + ret void +} + +; FUNC-LABEL: {{^}}store_local_v2i32: +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG-NOT: LDS_WRITE + +; CM: LDS_WRITE +; CM: LDS_WRITE +; CM-NOT: LDS_WRITE + +; GCN: ds_write_b64 +define void @store_local_v2i32(<2 x i32> addrspace(3)* %out, <2 x i32> %in) { +entry: + store <2 x i32> %in, <2 x i32> addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_v4i32: +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE + +; CM: LDS_WRITE +; CM: LDS_WRITE +; CM: LDS_WRITE +; CM: LDS_WRITE + +; GCN: ds_write2_b64 +define void @store_local_v4i32(<4 x i32> addrspace(3)* %out, <4 x i32> %in) { +entry: + store <4 x i32> %in, <4 x i32> addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_v4i32_align4: +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE +; EG: LDS_WRITE + +; CM: LDS_WRITE +; CM: LDS_WRITE +; CM: LDS_WRITE +; CM: LDS_WRITE + +; GCN: ds_write2_b32 +; GCN: ds_write2_b32 +define void @store_local_v4i32_align4(<4 x i32> addrspace(3)* %out, <4 x i32> %in) { +entry: + store <4 x i32> %in, <4 x i32> addrspace(3)* %out, align 4 + ret void +} + +; FUNC-LABEL: {{^}}store_local_i64_i8: +; EG: LDS_BYTE_WRITE +; GCN: ds_write_b8 +define void @store_local_i64_i8(i8 addrspace(3)* %out, i64 %in) { +entry: + %0 = trunc i64 %in to i8 + store i8 %0, i8 addrspace(3)* %out + ret void +} + +; FUNC-LABEL: {{^}}store_local_i64_i16: +; EG: LDS_SHORT_WRITE +; GCN: ds_write_b16 +define void @store_local_i64_i16(i16 addrspace(3)* %out, i64 %in) { +entry: + %0 = trunc i64 %in to i16 + store i16 %0, i16 addrspace(3)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/store-v3i32.ll b/test/CodeGen/AMDGPU/store-v3i32.ll deleted file mode 100644 index 33617b55ed64..000000000000 --- a/test/CodeGen/AMDGPU/store-v3i32.ll +++ /dev/null @@ -1,13 +0,0 @@ -; XFAIL: * -; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=SI < %s | FileCheck -check-prefix=SI %s -; RUN: llc -verify-machineinstrs -march=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=SI %s - -; 3 vectors have the same size and alignment as 4 vectors, so this -; should be done in a single store. - -; SI-LABEL: {{^}}store_v3i32: -; SI: buffer_store_dwordx4 -define void @store_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a) nounwind { - store <3 x i32> %a, <3 x i32> addrspace(1)* %out, align 16 - ret void -} diff --git a/test/CodeGen/AMDGPU/store-v3i64.ll b/test/CodeGen/AMDGPU/store-v3i64.ll index b4d7505e0a8a..03ae01afbcd0 100644 --- a/test/CodeGen/AMDGPU/store-v3i64.ll +++ b/test/CodeGen/AMDGPU/store-v3i64.ll @@ -46,8 +46,7 @@ define void @global_store_v3i64_unaligned(<3 x i64> addrspace(1)* %out, <3 x i64 } ; GCN-LABEL: {{^}}local_store_v3i64: -; GCN: ds_write_b64 -; GCN: ds_write_b64 +; GCN: ds_write2_b64 ; GCN: ds_write_b64 define void @local_store_v3i64(<3 x i64> addrspace(3)* %out, <3 x i64> %x) { store <3 x i64> %x, <3 x i64> addrspace(3)* %out, align 32 diff --git a/test/CodeGen/AMDGPU/store.r600.ll b/test/CodeGen/AMDGPU/store.r600.ll deleted file mode 100644 index 696fb033b5ec..000000000000 --- a/test/CodeGen/AMDGPU/store.r600.ll +++ /dev/null @@ -1,22 +0,0 @@ -; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck --check-prefix=EG %s - -; XXX: Merge this test into store.ll once it is supported on SI - -; v4i32 store -; EG: {{^}}store_v4i32: -; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XYZW, T[0-9]+\.X}}, 1 - -define void @store_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { - %1 = load <4 x i32>, <4 x i32> addrspace(1) * %in - store <4 x i32> %1, <4 x i32> addrspace(1)* %out - ret void -} - -; v4f32 store -; EG: {{^}}store_v4f32: -; EG: MEM_RAT_CACHELESS STORE_RAW T{{[0-9]+\.XYZW, T[0-9]+\.X}}, 1 -define void @store_v4f32(<4 x float> addrspace(1)* %out, <4 x float> addrspace(1)* %in) { - %1 = load <4 x float>, <4 x float> addrspace(1) * %in - store <4 x float> %1, <4 x float> addrspace(1)* %out - ret void -} diff --git a/test/CodeGen/AMDGPU/sub.i16.ll b/test/CodeGen/AMDGPU/sub.i16.ll new file mode 100644 index 000000000000..065bac3488d2 --- /dev/null +++ b/test/CodeGen/AMDGPU/sub.i16.ll @@ -0,0 +1,169 @@ +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefix=CI -check-prefix=GCN %s + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_subrev_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_sub_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = sub i16 %a, %b + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_constant: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xffffff85, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_sub_i16_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = sub i16 %a, 123 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_neg_constant: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0x34d, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_sub_i16_neg_constant(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = sub i16 %a, -845 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_inline_63: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: v_add_u16_e32 [[ADD:v[0-9]+]], 0xffffffc1, [[A]] +; VI-NEXT: buffer_store_short [[ADD]] +define void @v_test_sub_i16_inline_63(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = sub i16 %a, 63 + store i16 %add, i16 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_zext_to_i32: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_subrev_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: buffer_store_dword [[ADD]] +define void @v_test_sub_i16_zext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = sub i16 %a, %b + %ext = zext i16 %add to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_zext_to_i64: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI-DAG: v_subrev_u16_e32 v[[ADD:[0-9]+]], [[B]], [[A]] +; VI-DAG: v_mov_b32_e32 v[[VZERO:[0-9]+]], 0 +; VI: buffer_store_dwordx2 v{{\[}}[[ADD]]:[[VZERO]]{{\]}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0{{$}} +define void @v_test_sub_i16_zext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i64, i64 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %b = load volatile i16, i16 addrspace(1)* %gep.in1 + %add = sub i16 %a, %b + %ext = zext i16 %add to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i32: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_subrev_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: v_bfe_i32 [[SEXT:v[0-9]+]], [[ADD]], 0, 16 +; VI-NEXT: buffer_store_dword [[SEXT]] +define void @v_test_sub_i16_sext_to_i32(i32 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load i16, i16 addrspace(1)* %gep.in0 + %b = load i16, i16 addrspace(1)* %gep.in1 + %add = sub i16 %a, %b + %ext = sext i16 %add to i32 + store i32 %ext, i32 addrspace(1)* %out + ret void +} + +; FIXME: Need to handle non-uniform case for function below (load without gep). +; GCN-LABEL: {{^}}v_test_sub_i16_sext_to_i64: +; VI: flat_load_ushort [[A:v[0-9]+]] +; VI: flat_load_ushort [[B:v[0-9]+]] +; VI: v_subrev_u16_e32 [[ADD:v[0-9]+]], [[B]], [[A]] +; VI-NEXT: v_bfe_i32 v[[LO:[0-9]+]], [[ADD]], 0, 16 +; VI-NEXT: v_ashrrev_i32_e32 v[[HI:[0-9]+]], 31, v[[LO]] +; VI-NEXT: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}} +define void @v_test_sub_i16_sext_to_i64(i64 addrspace(1)* %out, i16 addrspace(1)* %in0, i16 addrspace(1)* %in1) #1 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i64, i64 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %gep.in1 = getelementptr inbounds i16, i16 addrspace(1)* %in1, i32 %tid + %a = load i16, i16 addrspace(1)* %gep.in0 + %b = load i16, i16 addrspace(1)* %gep.in1 + %add = sub i16 %a, %b + %ext = sext i16 %add to i64 + store i64 %ext, i64 addrspace(1)* %out + ret void +} + +@lds = addrspace(3) global [512 x i32] undef, align 4 + +; GCN-LABEL: {{^}}v_test_sub_i16_constant_commute: +; VI: v_subrev_u16_e32 v{{[0-9]+}}, 0x800, v{{[0-9]+}} +; CI: v_subrev_i32_e32 v{{[0-9]+}}, vcc, 0x800, v{{[0-9]+}} +define void @v_test_sub_i16_constant_commute(i16 addrspace(1)* %out, i16 addrspace(1)* %in0) #1 { + %size = call i32 @llvm.amdgcn.groupstaticsize() + %size.trunc = trunc i32 %size to i16 + call void asm sideeffect "; $0", "v"([512 x i32] addrspace(3)* @lds) + %tid = call i32 @llvm.amdgcn.workitem.id.x() + %gep.out = getelementptr inbounds i16, i16 addrspace(1)* %out, i32 %tid + %gep.in0 = getelementptr inbounds i16, i16 addrspace(1)* %in0, i32 %tid + %a = load volatile i16, i16 addrspace(1)* %gep.in0 + %add = sub i16 %a, %size.trunc + store i16 %add, i16 addrspace(1)* %out + ret void +} + +declare i32 @llvm.amdgcn.workitem.id.x() #0 +declare i32 @llvm.amdgcn.groupstaticsize() #0 + +attributes #0 = { nounwind readnone } +attributes #1 = { nounwind } diff --git a/test/CodeGen/AMDGPU/sub.ll b/test/CodeGen/AMDGPU/sub.ll index 5a026cdf2990..5816345098af 100644 --- a/test/CodeGen/AMDGPU/sub.ll +++ b/test/CodeGen/AMDGPU/sub.ll @@ -54,6 +54,46 @@ define void @test_sub_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1) ret void } +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +define void @test_sub_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) { + %b_ptr = getelementptr i16, i16 addrspace(1)* %in, i16 1 + %a = load i16, i16 addrspace(1)* %in + %b = load i16, i16 addrspace(1)* %b_ptr + %result = sub i16 %a, %b + store i16 %result, i16 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_sub_v2i16: + +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} + +define void @test_sub_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> addrspace(1)* %in) { + %b_ptr = getelementptr <2 x i16>, <2 x i16> addrspace(1)* %in, i16 1 + %a = load <2 x i16>, <2 x i16> addrspace(1) * %in + %b = load <2 x i16>, <2 x i16> addrspace(1) * %b_ptr + %result = sub <2 x i16> %a, %b + store <2 x i16> %result, <2 x i16> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}test_sub_v4i16: + +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} +; VI: v_sub_i16_e32 v{{[0-9]+, v[0-9]+, v[0-9]+}} + +define void @test_sub_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in) { + %b_ptr = getelementptr <4 x i16>, <4 x i16> addrspace(1)* %in, i16 1 + %a = load <4 x i16>, <4 x i16> addrspace(1) * %in + %b = load <4 x i16>, <4 x i16> addrspace(1) * %b_ptr + %result = sub <4 x i16> %a, %b + store <4 x i16> %result, <4 x i16> addrspace(1)* %out + ret void +} + ; FUNC-LABEL: {{^}}s_sub_i64: ; SI: s_sub_u32 ; SI: s_subb_u32 diff --git a/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll index 4b6f65a77b9a..72a1f1e25b30 100644 --- a/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll +++ b/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll @@ -8,7 +8,7 @@ target triple="amdgcn--" ; CHECK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb ; CHECK-NEXT: s_waitcnt lgkmcnt(0) ; CHECK: v_mbcnt_lo_u32_b32_e64 -; CHECK-NEXT: v_cmp_eq_i32_e32 vcc, 0, v0 +; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 ; CHECK-NEXT: s_and_saveexec_b64 s[2:3], vcc ; CHECK-NEXT: s_xor_b64 s[2:3], exec, s[2:3] ; BB0_1: diff --git a/test/CodeGen/AMDGPU/subreg-intervals.mir b/test/CodeGen/AMDGPU/subreg-intervals.mir new file mode 100644 index 000000000000..c4e00215159b --- /dev/null +++ b/test/CodeGen/AMDGPU/subreg-intervals.mir @@ -0,0 +1,51 @@ +# RUN: llc -march=amdgcn -run-pass liveintervals -debug-only=regalloc -o /dev/null %s 2>&1 | FileCheck %s +# REQUIRES: asserts + +# CHECK: INTERVALS +# CHECK: vreg0 +# CHECK-LABEL: Machine code for function test0: + +# CHECK: INTERVALS +# CHECK: vreg0 +# CHECK-LABEL: Machine code for function test1: + +--- | + define void @test0() { ret void } + define void @test1() { ret void } +... +--- +name: test0 +registers: + - { id: 0, class: sreg_64 } +body: | + bb.0: + S_NOP 0, implicit-def %0 + S_NOP 0, implicit %0 + + S_NOP 0, implicit-def undef %0.sub0 + S_NOP 0, implicit %0 +... +--- +name: test1 +registers: + - { id: 0, class: sreg_64 } +body: | + bb.0: + successors: %bb.1, %bb.2 + S_CBRANCH_VCCNZ %bb.1, implicit undef %vcc + S_BRANCH %bb.2 + + bb.1: + successors: %bb.3 + S_NOP 0, implicit-def undef %0.sub0 + S_BRANCH %bb.3 + + bb.2: + successors: %bb.3 + S_NOP 0, implicit-def %0 + S_BRANCH %bb.3 + + bb.3: + S_NOP 0 + S_NOP 0, implicit %0 +... diff --git a/test/CodeGen/AMDGPU/target-cpu.ll b/test/CodeGen/AMDGPU/target-cpu.ll index c1662acbf2a0..cf80ff3f4c83 100644 --- a/test/CodeGen/AMDGPU/target-cpu.ll +++ b/test/CodeGen/AMDGPU/target-cpu.ll @@ -108,5 +108,5 @@ attributes #1 = { nounwind readnone } attributes #2 = { nounwind "target-cpu"="tahiti" } attributes #3 = { nounwind "target-cpu"="bonaire" } attributes #4 = { nounwind "target-cpu"="fiji" } -attributes #5 = { nounwind "target-features"="+promote-alloca" "amdgpu-max-waves-per-eu"="3" } -attributes #6 = { nounwind "target-features"="-promote-alloca" "amdgpu-max-waves-per-eu"="3" } +attributes #5 = { nounwind "target-features"="+promote-alloca" "amdgpu-waves-per-eu"="1,3" } +attributes #6 = { nounwind "target-features"="-promote-alloca" "amdgpu-waves-per-eu"="1,3" } diff --git a/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll b/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll index 9e2373c55e35..f7aa4bc2c6dc 100644 --- a/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll +++ b/test/CodeGen/AMDGPU/trunc-bitcast-vector.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck --check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck --check-prefix=VI %s ; CHECK-LABEL: {{^}}trunc_i64_bitcast_v2i32: ; CHECK: buffer_load_dword v @@ -47,7 +47,12 @@ define void @trunc_i16_bitcast_v2i16(i16 addrspace(1)* %out, <2 x i16> addrspace } ; CHECK-LABEL: {{^}}trunc_i16_bitcast_v4i16: -; CHECK: buffer_load_dword [[VAL:v[0-9]+]] +; FIXME We need to teach the dagcombiner to reduce load width for: +; t21: v2i32,ch = load<LD8[%in(addrspace=1)]> t12, t10, undef:i64 +; t23: i64 = bitcast t21 +; t30: i16 = truncate t23 +; SI: buffer_load_dword v[[VAL:[0-9]+]] +; VI: buffer_load_dwordx2 v{{\[}}[[VAL:[0-9]+]] ; CHECK: buffer_store_short [[VAL]] define void @trunc_i16_bitcast_v4i16(i16 addrspace(1)* %out, <4 x i16> addrspace(1)* %in) { %ld = load <4 x i16>, <4 x i16> addrspace(1)* %in diff --git a/test/CodeGen/AMDGPU/trunc-cmp-constant.ll b/test/CodeGen/AMDGPU/trunc-cmp-constant.ll index 6d820dbd2692..b065fb266cec 100644 --- a/test/CodeGen/AMDGPU/trunc-cmp-constant.ll +++ b/test/CodeGen/AMDGPU/trunc-cmp-constant.ll @@ -1,10 +1,12 @@ ; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s +declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone + ; FUNC-LABEL {{^}}sextload_i1_to_i32_trunc_cmp_eq_0: ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]] ; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]] -; SI: v_cmp_eq_i32_e32 vcc, 0, [[TMP]]{{$}} +; SI: v_cmp_eq_u32_e32 vcc, 0, [[TMP]]{{$}} ; SI: v_cndmask_b32_e64 ; SI: buffer_store_byte define void @sextload_i1_to_i32_trunc_cmp_eq_0(i1 addrspace(1)* %out, i1 addrspace(1)* %in) nounwind { @@ -19,7 +21,7 @@ define void @sextload_i1_to_i32_trunc_cmp_eq_0(i1 addrspace(1)* %out, i1 addrspa ; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_eq_0: ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]] ; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]] -; SI: v_cmp_eq_i32_e32 vcc, 1, [[TMP]]{{$}} +; SI: v_cmp_eq_u32_e32 vcc, 1, [[TMP]]{{$}} ; SI-NEXT: s_xor_b64 [[NEG:s\[[0-9]+:[0-9]+\]]], vcc, -1 ; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[NEG]] ; SI: buffer_store_byte [[RESULT]] @@ -116,7 +118,7 @@ define void @sextload_i1_to_i32_trunc_cmp_ne_1(i1 addrspace(1)* %out, i1 addrspa ; FUNC-LABEL: {{^}}zextload_i1_to_i32_trunc_cmp_ne_1: ; SI: buffer_load_ubyte [[LOAD:v[0-9]+]] ; SI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]] -; SI: v_cmp_eq_i32_e32 vcc, 1, [[TMP]]{{$}} +; SI: v_cmp_eq_u32_e32 vcc, 1, [[TMP]]{{$}} ; SI-NEXT: s_xor_b64 [[NEG:s\[[0-9]+:[0-9]+\]]], vcc, -1 ; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[NEG]] ; SI: buffer_store_byte [[RESULT]] @@ -132,7 +134,7 @@ define void @zextload_i1_to_i32_trunc_cmp_ne_1(i1 addrspace(1)* %out, i1 addrspa ; FUNC-LABEL: {{^}}sextload_i1_to_i32_trunc_cmp_ne_neg1: ; XSI: buffer_load_ubyte [[LOAD:v[0-9]+]] ; XSI: v_and_b32_e32 [[TMP:v[0-9]+]], 1, [[LOAD]] -; XSI: v_cmp_eq_i32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], [[TMP]], 0{{$}} +; XSI: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]+\]]], [[TMP]], 0{{$}} ; XSI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CMP0]] ; XSI-NEXT: buffer_store_byte [[RESULT]] define void @sextload_i1_to_i32_trunc_cmp_ne_neg1(i1 addrspace(1)* %out, i1 addrspace(1)* %in) nounwind { @@ -154,13 +156,16 @@ define void @zextload_i1_to_i32_trunc_cmp_ne_neg1(i1 addrspace(1)* %out, i1 addr ret void } +; FIXME: Need to handle non-uniform case for function below (load without gep). ; FUNC-LABEL: {{^}}masked_load_i1_to_i32_trunc_cmp_ne_neg1: -; SI: buffer_load_sbyte [[LOAD:v[0-9]+]] -; SI: v_cmp_ne_i32_e32 vcc, -1, [[LOAD]]{{$}} +; SI: {{buffer|flat}}_load_sbyte [[LOAD:v[0-9]+]] +; SI: v_cmp_ne_u32_e32 vcc, -1, [[LOAD]]{{$}} ; SI-NEXT: v_cndmask_b32_e64 -; SI: buffer_store_byte +; SI: {{buffer|flat}}_store_byte define void @masked_load_i1_to_i32_trunc_cmp_ne_neg1(i1 addrspace(1)* %out, i8 addrspace(1)* %in) nounwind { - %load = load i8, i8 addrspace(1)* %in + %tid.x = call i32 @llvm.amdgcn.workitem.id.x() + %in.ptr = getelementptr i8, i8 addrspace(1)* %in, i32 %tid.x + %load = load i8, i8 addrspace(1)* %in.ptr %masked = and i8 %load, 255 %ext = sext i8 %masked to i32 %cmp = icmp ne i32 %ext, -1 diff --git a/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll b/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll index c29872beef86..03b8af0610d7 100644 --- a/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll +++ b/test/CodeGen/AMDGPU/trunc-store-f64-to-f16.ll @@ -1,5 +1,4 @@ -; XFAIL: * -; RUN: llc -march=amdgcn -mcpu=SI < %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s ; GCN-LABEL: {{^}}global_truncstore_f64_to_f16: ; GCN: s_endpgm diff --git a/test/CodeGen/AMDGPU/trunc-store-i1.ll b/test/CodeGen/AMDGPU/trunc-store-i1.ll index b71a838b62cd..b1e879f26301 100644 --- a/test/CodeGen/AMDGPU/trunc-store-i1.ll +++ b/test/CodeGen/AMDGPU/trunc-store-i1.ll @@ -21,13 +21,20 @@ define void @global_truncstore_i64_to_i1(i1 addrspace(1)* %out, i64 %val) nounwi ret void } -; SI-LABEL: {{^}}global_truncstore_i16_to_i1: +; SI-LABEL: {{^}}s_arg_global_truncstore_i16_to_i1: ; SI: s_load_dword [[LOAD:s[0-9]+]], ; SI: s_and_b32 [[SREG:s[0-9]+]], [[LOAD]], 1 ; SI: v_mov_b32_e32 [[VREG:v[0-9]+]], [[SREG]] ; SI: buffer_store_byte [[VREG]], -define void @global_truncstore_i16_to_i1(i1 addrspace(1)* %out, i16 %val) nounwind { +define void @s_arg_global_truncstore_i16_to_i1(i1 addrspace(1)* %out, i16 %val) nounwind { %trunc = trunc i16 %val to i1 store i1 %trunc, i1 addrspace(1)* %out, align 1 ret void } +; SI-LABEL: {{^}}global_truncstore_i16_to_i1: +define void @global_truncstore_i16_to_i1(i1 addrspace(1)* %out, i16 %val0, i16 %val1) nounwind { + %add = add i16 %val0, %val1 + %trunc = trunc i16 %add to i1 + store i1 %trunc, i1 addrspace(1)* %out, align 1 + ret void +} diff --git a/test/CodeGen/AMDGPU/trunc.ll b/test/CodeGen/AMDGPU/trunc.ll index dbd07fee6bbe..63bb447df2fd 100644 --- a/test/CodeGen/AMDGPU/trunc.ll +++ b/test/CodeGen/AMDGPU/trunc.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=amdgcn -mcpu=SI -verify-machineinstrs< %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI %s ; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s declare i32 @llvm.r600.read.tidig.x() nounwind readnone @@ -35,11 +35,11 @@ define void @trunc_load_shl_i64(i32 addrspace(1)* %out, i64 %a) { ; SI: s_load_dwordx2 s{{\[}}[[LO_SREG:[0-9]+]]:{{[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0xd ; SI: s_lshl_b64 s{{\[}}[[LO_SHL:[0-9]+]]:{{[0-9]+\]}}, s{{\[}}[[LO_SREG]]:{{[0-9]+\]}}, 2 ; SI: s_add_u32 s[[LO_SREG2:[0-9]+]], s[[LO_SHL]], +; SI: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], s[[LO_SREG2]] ; SI: s_addc_u32 +; SI: buffer_store_dword v[[LO_VREG]], ; SI: v_mov_b32_e32 -; SI: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], s[[LO_SREG2]] ; SI: v_mov_b32_e32 -; SI: buffer_store_dword v[[LO_VREG]], define void @trunc_shl_i64(i64 addrspace(1)* %out2, i32 addrspace(1)* %out, i64 %a) { %aa = add i64 %a, 234 ; Prevent shrinking store. %b = shl i64 %aa, 2 @@ -51,7 +51,7 @@ define void @trunc_shl_i64(i64 addrspace(1)* %out2, i32 addrspace(1)* %out, i64 ; SI-LABEL: {{^}}trunc_i32_to_i1: ; SI: v_and_b32_e32 v{{[0-9]+}}, 1, v{{[0-9]+}} -; SI: v_cmp_eq_i32 +; SI: v_cmp_eq_u32 define void @trunc_i32_to_i1(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) { %a = load i32, i32 addrspace(1)* %ptr, align 4 %trunc = trunc i32 %a to i1 @@ -62,7 +62,7 @@ define void @trunc_i32_to_i1(i32 addrspace(1)* %out, i32 addrspace(1)* %ptr) { ; SI-LABEL: {{^}}sgpr_trunc_i32_to_i1: ; SI: s_and_b32 s{{[0-9]+}}, 1, s{{[0-9]+}} -; SI: v_cmp_eq_i32 +; SI: v_cmp_eq_u32 define void @sgpr_trunc_i32_to_i1(i32 addrspace(1)* %out, i32 %a) { %trunc = trunc i32 %a to i1 %result = select i1 %trunc, i32 1, i32 0 @@ -73,7 +73,7 @@ define void @sgpr_trunc_i32_to_i1(i32 addrspace(1)* %out, i32 %a) { ; SI-LABEL: {{^}}s_trunc_i64_to_i1: ; SI: s_load_dwordx2 s{{\[}}[[SLO:[0-9]+]]:{{[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0xb ; SI: s_and_b32 [[MASKED:s[0-9]+]], 1, s[[SLO]] -; SI: v_cmp_eq_i32_e64 s{{\[}}[[VLO:[0-9]+]]:[[VHI:[0-9]+]]], 1, [[MASKED]] +; SI: v_cmp_eq_u32_e64 s{{\[}}[[VLO:[0-9]+]]:[[VHI:[0-9]+]]], [[MASKED]], 1{{$}} ; SI: v_cndmask_b32_e64 {{v[0-9]+}}, -12, 63, s{{\[}}[[VLO]]:[[VHI]]] define void @s_trunc_i64_to_i1(i32 addrspace(1)* %out, i64 %x) { %trunc = trunc i64 %x to i1 @@ -85,7 +85,7 @@ define void @s_trunc_i64_to_i1(i32 addrspace(1)* %out, i64 %x) { ; SI-LABEL: {{^}}v_trunc_i64_to_i1: ; SI: buffer_load_dwordx2 v{{\[}}[[VLO:[0-9]+]]:{{[0-9]+\]}} ; SI: v_and_b32_e32 [[MASKED:v[0-9]+]], 1, v[[VLO]] -; SI: v_cmp_eq_i32_e32 vcc, 1, [[MASKED]] +; SI: v_cmp_eq_u32_e32 vcc, 1, [[MASKED]] ; SI: v_cndmask_b32_e64 {{v[0-9]+}}, -12, 63, vcc define void @v_trunc_i64_to_i1(i32 addrspace(1)* %out, i64 addrspace(1)* %in) { %tid = call i32 @llvm.r600.read.tidig.x() nounwind readnone diff --git a/test/CodeGen/AMDGPU/udiv.ll b/test/CodeGen/AMDGPU/udiv.ll index f72c22095e4a..02383a972056 100644 --- a/test/CodeGen/AMDGPU/udiv.ll +++ b/test/CodeGen/AMDGPU/udiv.ll @@ -145,3 +145,16 @@ define void @v_udiv_i24(i32 addrspace(1)* %out, i24 addrspace(1)* %in) { store i32 %result.ext, i32 addrspace(1)* %out ret void } + +; FUNC-LABEL: @scalarize_mulhu_4xi32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 +; SI: v_mul_hi_u32 + +define void @scalarize_mulhu_4xi32(<4 x i32> addrspace(1)* nocapture readonly %in, <4 x i32> addrspace(1)* nocapture %out) { + %1 = load <4 x i32>, <4 x i32> addrspace(1)* %in, align 16 + %2 = udiv <4 x i32> %1, <i32 53668, i32 53668, i32 53668, i32 53668> + store <4 x i32> %2, <4 x i32> addrspace(1)* %out, align 16 + ret void +} diff --git a/test/CodeGen/AMDGPU/udivrem.ll b/test/CodeGen/AMDGPU/udivrem.ll index 268f3c764d6e..078df5307334 100644 --- a/test/CodeGen/AMDGPU/udivrem.ll +++ b/test/CodeGen/AMDGPU/udivrem.ll @@ -38,16 +38,16 @@ ; SI: v_cndmask_b32_e64 ; SI: v_mul_hi_u32 [[Quotient:v[0-9]+]] ; SI: v_mul_lo_i32 [[Num_S_Remainder:v[0-9]+]] +; SI-DAG: v_add_i32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]] ; SI-DAG: v_sub_i32_e32 [[Remainder:v[0-9]+]], vcc, {{[vs][0-9]+}}, [[Num_S_Remainder]] ; SI-DAG: v_cndmask_b32_e64 ; SI-DAG: v_cndmask_b32_e64 -; SI: v_and_b32_e32 [[Tmp1:v[0-9]+]] -; SI-DAG: v_add_i32_e32 [[Quotient_A_One:v[0-9]+]], vcc, 1, [[Quotient]] ; SI-DAG: v_subrev_i32_e32 [[Quotient_S_One:v[0-9]+]], +; SI-DAG: v_subrev_i32_e32 [[Remainder_S_Den:v[0-9]+]], +; SI: v_and_b32_e32 [[Tmp1:v[0-9]+]] ; SI-DAG: v_cndmask_b32_e64 ; SI-DAG: v_cndmask_b32_e64 ; SI-DAG: v_add_i32_e32 [[Remainder_A_Den:v[0-9]+]], -; SI-DAG: v_subrev_i32_e32 [[Remainder_S_Den:v[0-9]+]], ; SI-DAG: v_cndmask_b32_e64 ; SI-DAG: v_cndmask_b32_e64 ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/test/CodeGen/AMDGPU/uint_to_fp.f64.ll index b36ce6b8d6ca..a4e18ebc9120 100644 --- a/test/CodeGen/AMDGPU/uint_to_fp.f64.ll +++ b/test/CodeGen/AMDGPU/uint_to_fp.f64.ll @@ -74,7 +74,7 @@ define void @s_uint_to_fp_v4i32_to_v4f64(<4 x double> addrspace(1)* %out, <4 x i ; uses an SGPR (implicit vcc). ; SI-LABEL: {{^}}uint_to_fp_i1_to_f64: -; SI-DAG: v_cmp_eq_i32_e64 vcc +; SI-DAG: v_cmp_eq_u32_e64 vcc ; SI-DAG: v_cndmask_b32_e32 v[[SEL:[0-9]+]], 0, v{{[0-9]+}} ; SI-DAG: v_mov_b32_e32 v[[ZERO:[0-9]+]], 0{{$}} ; SI: buffer_store_dwordx2 v{{\[}}[[ZERO]]:[[SEL]]{{\]}} diff --git a/test/CodeGen/AMDGPU/uint_to_fp.i64.ll b/test/CodeGen/AMDGPU/uint_to_fp.i64.ll index 27c41e41a0e7..22d563d40b74 100644 --- a/test/CodeGen/AMDGPU/uint_to_fp.i64.ll +++ b/test/CodeGen/AMDGPU/uint_to_fp.i64.ll @@ -3,6 +3,37 @@ ; FIXME: This should be merged with uint_to_fp.ll, but s_uint_to_fp_v2i64 crashes on r600 +; FUNC-LABEL: {{^}}s_uint_to_fp_i64_to_f16: +define void @s_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 %in) #0 { + %result = uitofp i64 %in to half + store half %result, half addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_uint_to_fp_i64_to_f16: +; GCN: {{buffer|flat}}_load_dwordx2 + +; GCN: v_ffbh_u32 +; GCN: v_ffbh_u32 +; GCN: v_cndmask +; GCN: v_cndmask + +; GCN-DAG: v_cmp_eq_u64 +; GCN-DAG: v_cmp_lt_u64 + +; GCN: v_add_i32_e32 [[VR:v[0-9]+]] +; GCN: v_cvt_f16_f32_e32 [[VR_F16:v[0-9]+]], [[VR]] +; GCN: {{buffer|flat}}_store_short {{.*}}[[VR_F16]] +define void @v_uint_to_fp_i64_to_f16(half addrspace(1)* %out, i64 addrspace(1)* %in) #0 { + %tid = call i32 @llvm.r600.read.tidig.x() + %in.gep = getelementptr i64, i64 addrspace(1)* %in, i32 %tid + %out.gep = getelementptr half, half addrspace(1)* %out, i32 %tid + %val = load i64, i64 addrspace(1)* %in.gep + %result = uitofp i64 %val to half + store half %result, half addrspace(1)* %out.gep + ret void +} + ; FUNC-LABEL: {{^}}s_uint_to_fp_i64_to_f32: define void @s_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 %in) #0 { %result = uitofp i64 %in to float @@ -18,7 +49,7 @@ define void @s_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 %in) #0 { ; GCN: v_cndmask ; GCN: v_cndmask -; GCN-DAG: v_cmp_eq_i64 +; GCN-DAG: v_cmp_eq_u64 ; GCN-DAG: v_cmp_lt_u64 ; GCN: v_add_i32_e32 [[VR:v[0-9]+]] @@ -33,15 +64,15 @@ define void @v_uint_to_fp_i64_to_f32(float addrspace(1)* %out, i64 addrspace(1)* ret void } -; FUNC-LABEL: {{^}}s_uint_to_fp_v2i64: -define void @s_uint_to_fp_v2i64(<2 x float> addrspace(1)* %out, <2 x i64> %in) #0{ +; FUNC-LABEL: {{^}}s_uint_to_fp_v2i64_to_v2f32: +define void @s_uint_to_fp_v2i64_to_v2f32(<2 x float> addrspace(1)* %out, <2 x i64> %in) #0{ %result = uitofp <2 x i64> %in to <2 x float> store <2 x float> %result, <2 x float> addrspace(1)* %out ret void } -; FUNC-LABEL: {{^}}v_uint_to_fp_v4i64: -define void @v_uint_to_fp_v4i64(<4 x float> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { +; FUNC-LABEL: {{^}}v_uint_to_fp_v4i64_to_v4f32: +define void @v_uint_to_fp_v4i64_to_v4f32(<4 x float> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { %tid = call i32 @llvm.r600.read.tidig.x() %in.gep = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i32 %tid %out.gep = getelementptr <4 x float>, <4 x float> addrspace(1)* %out, i32 %tid @@ -51,6 +82,24 @@ define void @v_uint_to_fp_v4i64(<4 x float> addrspace(1)* %out, <4 x i64> addrsp ret void } +; FUNC-LABEL: {{^}}s_uint_to_fp_v2i64_to_v2f16: +define void @s_uint_to_fp_v2i64_to_v2f16(<2 x half> addrspace(1)* %out, <2 x i64> %in) #0{ + %result = uitofp <2 x i64> %in to <2 x half> + store <2 x half> %result, <2 x half> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}v_uint_to_fp_v4i64_to_v4f16: +define void @v_uint_to_fp_v4i64_to_v4f16(<4 x half> addrspace(1)* %out, <4 x i64> addrspace(1)* %in) #0 { + %tid = call i32 @llvm.r600.read.tidig.x() + %in.gep = getelementptr <4 x i64>, <4 x i64> addrspace(1)* %in, i32 %tid + %out.gep = getelementptr <4 x half>, <4 x half> addrspace(1)* %out, i32 %tid + %value = load <4 x i64>, <4 x i64> addrspace(1)* %in.gep + %result = uitofp <4 x i64> %value to <4 x half> + store <4 x half> %result, <4 x half> addrspace(1)* %out.gep + ret void +} + declare i32 @llvm.r600.read.tidig.x() #1 attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/uint_to_fp.ll b/test/CodeGen/AMDGPU/uint_to_fp.ll index 0c3d54cf0d09..aae055083663 100644 --- a/test/CodeGen/AMDGPU/uint_to_fp.ll +++ b/test/CodeGen/AMDGPU/uint_to_fp.ll @@ -77,7 +77,7 @@ define void @v_uint_to_fp_v4i32(<4 x float> addrspace(1)* %out, <4 x i32> addrsp } ; FUNC-LABEL: {{^}}s_uint_to_fp_i1_to_f32: -; SI: v_cmp_eq_i32_e64 [[CMP:s\[[0-9]+:[0-9]\]]], +; SI: v_cmp_eq_u32_e64 [[CMP:s\[[0-9]+:[0-9]\]]], ; SI-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0, [[CMP]] ; SI: buffer_store_dword [[RESULT]], ; SI: s_endpgm @@ -101,7 +101,7 @@ define void @s_uint_to_fp_i1_to_f32_load(float addrspace(1)* %out, i1 %in) #0 { ; FUNC-LABEL: {{^}}v_uint_to_fp_i1_f32_load: ; SI: {{buffer|flat}}_load_ubyte ; SI: v_and_b32_e32 {{v[0-9]+}}, 1, {{v[0-9]+}} -; SI: v_cmp_eq_i32 +; SI: v_cmp_eq_u32 ; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1.0 ; SI: {{buffer|flat}}_store_dword {{.*}}[[RESULT]] ; SI: s_endpgm diff --git a/test/CodeGen/AMDGPU/uitofp.f16.ll b/test/CodeGen/AMDGPU/uitofp.f16.ll new file mode 100644 index 000000000000..73d9e8c9c222 --- /dev/null +++ b/test/CodeGen/AMDGPU/uitofp.f16.ll @@ -0,0 +1,83 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}uitofp_i16_to_f16 +; GCN: buffer_load_ushort v[[A_I16:[0-9]+]] +; SI: v_cvt_f32_u32_e32 v[[A_F32:[0-9]+]], v[[A_I16]] +; VI: v_cvt_f32_i32_e32 v[[A_F32:[0-9]+]], v[[A_I16]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_F32]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @uitofp_i16_to_f16( + half addrspace(1)* %r, + i16 addrspace(1)* %a) { +entry: + %a.val = load i16, i16 addrspace(1)* %a + %r.val = uitofp i16 %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}uitofp_i32_to_f16 +; GCN: buffer_load_dword v[[A_I32:[0-9]+]] +; GCN: v_cvt_f32_u32_e32 v[[A_I16:[0-9]+]], v[[A_I32]] +; GCN: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[A_I16]] +; GCN: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @uitofp_i32_to_f16( + half addrspace(1)* %r, + i32 addrspace(1)* %a) { +entry: + %a.val = load i32, i32 addrspace(1)* %a + %r.val = uitofp i32 %a.val to half + store half %r.val, half addrspace(1)* %r + ret void +} + +; f16 = uitofp i64 is in uint_to_fp.i64.ll + +; GCN-LABEL: {{^}}uitofp_v2i16_to_v2f16 +; GCN: buffer_load_dword +; SI: v_cvt_f32_u32_e32 +; SI: v_cvt_f32_u32_e32 +; VI: v_cvt_f32_i32_e32 +; VI: v_cvt_f32_i32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN-DAG: v_and_b32_e32 +; GCN-DAG: v_lshlrev_b32_e32 +; GCN-DAG: v_or_b32_e32 +; GCN: buffer_store_dword +; GCN: s_endpgm +define void @uitofp_v2i16_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x i16> addrspace(1)* %a) { +entry: + %a.val = load <2 x i16>, <2 x i16> addrspace(1)* %a + %r.val = uitofp <2 x i16> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}uitofp_v2i32_to_v2f16 +; GCN: buffer_load_dwordx2 +; GCN: v_cvt_f32_u32_e32 +; GCN: v_cvt_f32_u32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN: v_cvt_f16_f32_e32 +; GCN-DAG: v_and_b32_e32 +; GCN-DAG: v_lshlrev_b32_e32 +; GCN-DAG: v_or_b32_e32 +; GCN: buffer_store_dword +; GCN: s_endpgm +define void @uitofp_v2i32_to_v2f16( + <2 x half> addrspace(1)* %r, + <2 x i32> addrspace(1)* %a) { +entry: + %a.val = load <2 x i32>, <2 x i32> addrspace(1)* %a + %r.val = uitofp <2 x i32> %a.val to <2 x half> + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; f16 = uitofp i64 is in uint_to_fp.i64.ll diff --git a/test/CodeGen/AMDGPU/unaligned-load-store.ll b/test/CodeGen/AMDGPU/unaligned-load-store.ll index 129748afd938..453c981db1a5 100644 --- a/test/CodeGen/AMDGPU/unaligned-load-store.ll +++ b/test/CodeGen/AMDGPU/unaligned-load-store.ll @@ -552,4 +552,53 @@ define void @constant_align4_merge_load_2_i32(i32 addrspace(2)* %p, i32 addrspac ret void } +; SI-LABEL: {{^}}local_load_align1_v16i8: +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 +; SI: ds_read_u8 + +; SI: ScratchSize: 0{{$}} +define void @local_load_align1_v16i8(<16 x i8> addrspace(1)* %out, <16 x i8> addrspace(3)* %in) #0 { + %ld = load <16 x i8>, <16 x i8> addrspace(3)* %in, align 1 + store <16 x i8> %ld, <16 x i8> addrspace(1)* %out + ret void +} + +; SI-LABEL: {{^}}local_store_align1_v16i8: +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 +; SI: ds_write_b8 + +; SI: ScratchSize: 0{{$}} +define void @local_store_align1_v16i8(<16 x i8> addrspace(3)* %out) #0 { + store <16 x i8> zeroinitializer, <16 x i8> addrspace(3)* %out, align 1 + ret void +} + attributes #0 = { nounwind } diff --git a/test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll b/test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll index f09f73c38b4c..d96ee6d21ce8 100644 --- a/test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll +++ b/test/CodeGen/AMDGPU/unhandled-loop-condition-assertion.ll @@ -40,7 +40,7 @@ for.end: ; preds = %for.body, %entry ; COMMON-LABEL: {{^}}branch_false: ; SI: s_cbranch_vccnz -; SI: s_cbranch_vccnz +; SI: s_cbranch_scc1 ; SI: s_endpgm define void @branch_false(i8 addrspace(1)* nocapture %main, i32 %main_stride) #0 { entry: @@ -76,8 +76,8 @@ for.end: ; preds = %for.body, %entry } ; COMMON-LABEL: {{^}}branch_undef: -; SI: s_cbranch_vccnz -; SI: s_cbranch_vccnz +; SI: s_cbranch_scc1 +; SI: s_cbranch_scc1 ; SI: s_endpgm define void @branch_undef(i8 addrspace(1)* nocapture %main, i32 %main_stride) #0 { entry: diff --git a/test/CodeGen/AMDGPU/uniform-cfg.ll b/test/CodeGen/AMDGPU/uniform-cfg.ll index ac9e2b5f8432..a0060bd368be 100644 --- a/test/CodeGen/AMDGPU/uniform-cfg.ll +++ b/test/CodeGen/AMDGPU/uniform-cfg.ll @@ -1,16 +1,17 @@ -; RUN: llc < %s -march=amdgcn -mcpu=verde -verify-machineinstrs | FileCheck --check-prefix=SI %s -; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck --check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=verde -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -machine-sink-split-probability-threshold=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s -; SI-LABEL: {{^}}uniform_if_scc: -; SI-DAG: s_cmp_eq_i32 s{{[0-9]+}}, 0 -; SI-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0 -; SI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_scc: +; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0 +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 +; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] ; Fall-through to the else -; SI: v_mov_b32_e32 [[STORE_VAL]], 1 +; GCN: s_mov_b32 [[S_VAL]], 1 -; SI: [[IF_LABEL]]: -; SI: buffer_store_dword [[STORE_VAL]] +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] define void @uniform_if_scc(i32 %cond, i32 addrspace(1)* %out) { entry: %cmp0 = icmp eq i32 %cond, 0 @@ -28,19 +29,17 @@ done: ret void } -; SI-LABEL: {{^}}uniform_if_vcc: -; FIXME: We could use _e32 here if we re-used the 0 from [[STORE_VAL]], and -; also scheduled the write first. -; SI-DAG: v_cmp_eq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 0, s{{[0-9]+}} -; SI-DAG: s_and_b64 vcc, exec, [[COND]] -; SI-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0 -; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_vcc: +; GCN-DAG: v_cmp_eq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 +; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] ; Fall-through to the else -; SI: v_mov_b32_e32 [[STORE_VAL]], 1 +; GCN: s_mov_b32 [[S_VAL]], 1 -; SI: [[IF_LABEL]]: -; SI: buffer_store_dword [[STORE_VAL]] +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] define void @uniform_if_vcc(float %cond, i32 addrspace(1)* %out) { entry: %cmp0 = fcmp oeq float %cond, 0.0 @@ -58,16 +57,17 @@ done: ret void } -; SI-LABEL: {{^}}uniform_if_swap_br_targets_scc: -; SI-DAG: s_cmp_lg_i32 s{{[0-9]+}}, 0 -; SI-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0 -; SI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_swap_br_targets_scc: +; GCN-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 +; GCN: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] ; Fall-through to the else -; SI: v_mov_b32_e32 [[STORE_VAL]], 1 +; GCN: s_mov_b32 [[S_VAL]], 1 -; SI: [[IF_LABEL]]: -; SI: buffer_store_dword [[STORE_VAL]] +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] define void @uniform_if_swap_br_targets_scc(i32 %cond, i32 addrspace(1)* %out) { entry: %cmp0 = icmp eq i32 %cond, 0 @@ -85,19 +85,17 @@ done: ret void } -; SI-LABEL: {{^}}uniform_if_swap_br_targets_vcc: -; FIXME: We could use _e32 here if we re-used the 0 from [[STORE_VAL]], and -; also scheduled the write first. -; SI-DAG: v_cmp_neq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 0, s{{[0-9]+}} -; SI-DAG: s_and_b64 vcc, exec, [[COND]] -; SI-DAG: v_mov_b32_e32 [[STORE_VAL:v[0-9]+]], 0 -; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_swap_br_targets_vcc: +; GCN-DAG: v_cmp_neq_f32_e64 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], s{{[0-9]+}}, 0{{$}} +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 +; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] ; Fall-through to the else -; SI: v_mov_b32_e32 [[STORE_VAL]], 1 +; GCN: s_mov_b32 [[S_VAL]], 1 -; SI: [[IF_LABEL]]: -; SI: buffer_store_dword [[STORE_VAL]] +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] define void @uniform_if_swap_br_targets_vcc(float %cond, i32 addrspace(1)* %out) { entry: %cmp0 = fcmp oeq float %cond, 0.0 @@ -115,16 +113,16 @@ done: ret void } -; SI-LABEL: {{^}}uniform_if_move_valu: -; SI: v_add_f32_e32 [[CMP:v[0-9]+]] +; GCN-LABEL: {{^}}uniform_if_move_valu: +; GCN: v_add_f32_e32 [[CMP:v[0-9]+]] ; Using a floating-point value in an integer compare will cause the compare to ; be selected for the SALU and then later moved to the VALU. -; SI: v_cmp_ne_i32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 5, [[CMP]] -; SI: s_and_b64 vcc, exec, [[COND]] -; SI: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] -; SI: buffer_store_dword -; SI: [[ENDIF_LABEL]]: -; SI: s_endpgm +; GCN: v_cmp_ne_u32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 5, [[CMP]] +; GCN: s_and_b64 vcc, exec, [[COND]] +; GCN: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] +; GCN: buffer_store_dword +; GCN: [[ENDIF_LABEL]]: +; GCN: s_endpgm define void @uniform_if_move_valu(i32 addrspace(1)* %out, float %a) { entry: %a.0 = fadd float %a, 10.0 @@ -140,16 +138,16 @@ endif: ret void } -; SI-LABEL: {{^}}uniform_if_move_valu_commute: -; SI: v_add_f32_e32 [[CMP:v[0-9]+]] +; GCN-LABEL: {{^}}uniform_if_move_valu_commute: +; GCN: v_add_f32_e32 [[CMP:v[0-9]+]] ; Using a floating-point value in an integer compare will cause the compare to ; be selected for the SALU and then later moved to the VALU. -; SI: v_cmp_gt_u32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 6, [[CMP]] -; SI: s_and_b64 vcc, exec, [[COND]] -; SI: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] -; SI: buffer_store_dword -; SI: [[ENDIF_LABEL]]: -; SI: s_endpgm +; GCN: v_cmp_gt_u32_e32 [[COND:vcc|s\[[0-9]+:[0-9]+\]]], 6, [[CMP]] +; GCN: s_and_b64 vcc, exec, [[COND]] +; GCN: s_cbranch_vccnz [[ENDIF_LABEL:[0-9_A-Za-z]+]] +; GCN: buffer_store_dword +; GCN: [[ENDIF_LABEL]]: +; GCN: s_endpgm define void @uniform_if_move_valu_commute(i32 addrspace(1)* %out, float %a) { entry: %a.0 = fadd float %a, 10.0 @@ -166,18 +164,18 @@ endif: } -; SI-LABEL: {{^}}uniform_if_else_ret: -; SI: s_cmp_lg_i32 s{{[0-9]+}}, 0 -; SI-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_else_ret: +; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; GCN-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] -; SI: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; SI: buffer_store_dword [[TWO]] -; SI: s_endpgm +; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 +; GCN: buffer_store_dword [[TWO]] +; GCN: s_endpgm -; SI: {{^}}[[IF_LABEL]]: -; SI: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; SI: buffer_store_dword [[ONE]] -; SI: s_endpgm +; GCN: {{^}}[[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 +; GCN: buffer_store_dword [[ONE]] +; GCN: s_endpgm define void @uniform_if_else_ret(i32 addrspace(1)* nocapture %out, i32 %a) { entry: %cmp = icmp eq i32 %a, 0 @@ -195,22 +193,22 @@ if.end: ; preds = %if.else, %if.then ret void } -; SI-LABEL: {{^}}uniform_if_else: -; SI: s_cmp_lg_i32 s{{[0-9]+}}, 0 -; SI-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] +; GCN-LABEL: {{^}}uniform_if_else: +; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; GCN-NEXT: s_cbranch_scc0 [[IF_LABEL:[0-9_A-Za-z]+]] -; SI: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; SI: buffer_store_dword [[TWO]] -; SI: s_branch [[ENDIF_LABEL:[0-9_A-Za-z]+]] +; GCN: v_mov_b32_e32 [[IMM_REG:v[0-9]+]], 2 +; GCN: s_branch [[ENDIF_LABEL:[0-9_A-Za-z]+]] -; SI: [[IF_LABEL]]: -; SI: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; SI: buffer_store_dword [[ONE]] +; GCN: [[IF_LABEL]]: +; GCN-NEXT: v_mov_b32_e32 [[IMM_REG]], 1 -; SI: [[ENDIF_LABEL]]: -; SI: v_mov_b32_e32 [[THREE:v[0-9]+]], 3 -; SI: buffer_store_dword [[THREE]] -; SI: s_endpgm +; GCN-NEXT: [[ENDIF_LABEL]]: +; GCN: buffer_store_dword [[IMM_REG]] + +; GCN: v_mov_b32_e32 [[THREE:v[0-9]+]], 3 +; GCN: buffer_store_dword [[THREE]] +; GCN: s_endpgm define void @uniform_if_else(i32 addrspace(1)* nocapture %out0, i32 addrspace(1)* nocapture %out1, i32 %a) { entry: %cmp = icmp eq i32 %a, 0 @@ -229,12 +227,12 @@ if.end: ; preds = %if.else, %if.then ret void } -; SI-LABEL: {{^}}icmp_2_users: -; SI: s_cmp_lt_i32 s{{[0-9]+}}, 1 -; SI: s_cbranch_scc1 [[LABEL:[a-zA-Z0-9_]+]] -; SI: buffer_store_dword -; SI: [[LABEL]]: -; SI: s_endpgm +; GCN-LABEL: {{^}}icmp_2_users: +; GCN: s_cmp_lt_i32 s{{[0-9]+}}, 1 +; GCN: s_cbranch_scc1 [[LABEL:[a-zA-Z0-9_]+]] +; GCN: buffer_store_dword +; GCN: [[LABEL]]: +; GCN: s_endpgm define void @icmp_2_users(i32 addrspace(1)* %out, i32 %cond) { main_body: %0 = icmp sgt i32 %cond, 0 @@ -249,16 +247,15 @@ ENDIF: ; preds = %IF, %main_body ret void } -; SI-LABEL: {{^}}icmp_users_different_blocks: -; SI: s_load_dword [[COND:s[0-9]+]] -; SI: s_cmp_lt_i32 [[COND]], 1 -; SI: s_cbranch_scc1 [[EXIT:[A-Za-z0-9_]+]] -; SI: v_cmp_lt_i32_e64 [[MASK:s\[[0-9]+:[0-9]+\]]], 0, [[COND]] -; SI: s_and_b64 vcc, exec, [[MASK]] -; SI: s_cbranch_vccnz [[EXIT]] -; SI: buffer_store -; SI: {{^}}[[EXIT]]: -; SI: s_endpgm +; GCN-LABEL: {{^}}icmp_users_different_blocks: +; GCN: s_load_dword [[COND:s[0-9]+]] +; GCN: s_cmp_lt_i32 [[COND]], 1 +; GCN: s_cbranch_scc1 [[EXIT:[A-Za-z0-9_]+]] +; GCN: v_cmp_gt_i32_e64 vcc, [[COND]], 0{{$}} +; GCN: s_cbranch_vccnz [[EXIT]] +; GCN: buffer_store +; GCN: {{^}}[[EXIT]]: +; GCN: s_endpgm define void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, i32 addrspace(1)* %out) { bb: %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 @@ -281,12 +278,9 @@ bb9: ; preds = %bb8, %bb4 ; SI-LABEL: {{^}}uniform_loop: ; SI: {{^}}[[LOOP_LABEL:[A-Z0-9_a-z]+]]: -; FIXME: We need to teach SIFixSGPRCopies about uniform branches so we -; get s_add_i32 here. -; SI: v_add_i32_e32 [[I:v[0-9]+]], vcc, -1, v{{[0-9]+}} -; SI: v_cmp_ne_i32_e32 vcc, 0, [[I]] -; SI: s_and_b64 vcc, exec, vcc -; SI: s_cbranch_vccnz [[LOOP_LABEL]] +; SI: s_add_i32 [[I:s[0-9]+]], s{{[0-9]+}}, -1 +; SI: s_cmp_lg_u32 [[I]], 0 +; SI: s_cbranch_scc1 [[LOOP_LABEL]] ; SI: s_endpgm define void @uniform_loop(i32 addrspace(1)* %out, i32 %a) { entry: @@ -304,15 +298,15 @@ done: ; Test uniform and divergent. -; SI-LABEL: {{^}}uniform_inside_divergent: -; SI: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} -; SI: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc -; SI: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] -; SI: s_cbranch_execz [[ENDIF_LABEL:[0-9_A-Za-z]+]] -; SI: s_cmp_lg_i32 {{s[0-9]+}}, 0 -; SI: s_cbranch_scc1 [[ENDIF_LABEL]] -; SI: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; SI: buffer_store_dword [[ONE]] +; GCN-LABEL: {{^}}uniform_inside_divergent: +; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} +; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc +; GCN: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] +; GCN: s_cbranch_execz [[ENDIF_LABEL:[0-9_A-Za-z]+]] +; GCN: s_cmp_lg_u32 {{s[0-9]+}}, 0 +; GCN: s_cbranch_scc1 [[ENDIF_LABEL]] +; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 +; GCN: buffer_store_dword [[ONE]] define void @uniform_inside_divergent(i32 addrspace(1)* %out, i32 %cond) { entry: %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 @@ -332,16 +326,16 @@ endif: ret void } -; SI-LABEL: {{^}}divergent_inside_uniform: -; SI: s_cmp_lg_i32 s{{[0-9]+}}, 0 -; SI: s_cbranch_scc1 [[ENDIF_LABEL:[0-9_A-Za-z]+]] -; SI: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} -; SI: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc -; SI: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] -; SI: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; SI: buffer_store_dword [[ONE]] -; SI: [[ENDIF_LABEL]]: -; SI: s_endpgm +; GCN-LABEL: {{^}}divergent_inside_uniform: +; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; GCN: s_cbranch_scc1 [[ENDIF_LABEL:[0-9_A-Za-z]+]] +; GCN: v_cmp_gt_u32_e32 vcc, 16, v{{[0-9]+}} +; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc +; GCN: s_xor_b64 [[MASK1:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] +; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 +; GCN: buffer_store_dword [[ONE]] +; GCN: [[ENDIF_LABEL]]: +; GCN: s_endpgm define void @divergent_inside_uniform(i32 addrspace(1)* %out, i32 %cond) { entry: %u_cmp = icmp eq i32 %cond, 0 @@ -361,19 +355,19 @@ endif: ret void } -; SI-LABEL: {{^}}divergent_if_uniform_if: -; SI: v_cmp_eq_i32_e32 vcc, 0, v0 -; SI: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc -; SI: s_xor_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] -; SI: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 -; SI: buffer_store_dword [[ONE]] -; SI: s_or_b64 exec, exec, [[MASK]] -; SI: s_cmp_lg_i32 s{{[0-9]+}}, 0 -; SI: s_cbranch_scc1 [[EXIT:[A-Z0-9_]+]] -; SI: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 -; SI: buffer_store_dword [[TWO]] -; SI: [[EXIT]]: -; SI: s_endpgm +; GCN-LABEL: {{^}}divergent_if_uniform_if: +; GCN: v_cmp_eq_u32_e32 vcc, 0, v0 +; GCN: s_and_saveexec_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], vcc +; GCN: s_xor_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec, [[MASK]] +; GCN: v_mov_b32_e32 [[ONE:v[0-9]+]], 1 +; GCN: buffer_store_dword [[ONE]] +; GCN: s_or_b64 exec, exec, [[MASK]] +; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0 +; GCN: s_cbranch_scc1 [[EXIT:[A-Z0-9_]+]] +; GCN: v_mov_b32_e32 [[TWO:v[0-9]+]], 2 +; GCN: buffer_store_dword [[TWO]] +; GCN: [[EXIT]]: +; GCN: s_endpgm define void @divergent_if_uniform_if(i32 addrspace(1)* %out, i32 %cond) { entry: %tid = call i32 @llvm.amdgcn.workitem.id.x() #0 @@ -401,19 +395,19 @@ exit: ; the first, leaving an scc use in a different block than it was ; defed. -; SI-LABEL: {{^}}cse_uniform_condition_different_blocks: -; SI: s_load_dword [[COND:s[0-9]+]] -; SI: s_cmp_lt_i32 [[COND]], 1 -; SI: s_cbranch_scc1 BB[[FNNUM:[0-9]+]]_3 +; GCN-LABEL: {{^}}cse_uniform_condition_different_blocks: +; GCN: s_load_dword [[COND:s[0-9]+]] +; GCN: s_cmp_lt_i32 [[COND]], 1 +; GCN: s_cbranch_scc1 BB[[FNNUM:[0-9]+]]_3 -; SI: BB#1: -; SI-NOT: cmp -; SI: buffer_load_dword -; SI: buffer_store_dword -; SI: s_cbranch_scc1 BB[[FNNUM]]_3 +; GCN: BB#1: +; GCN-NOT: cmp +; GCN: buffer_load_dword +; GCN: buffer_store_dword +; GCN: s_cbranch_scc1 BB[[FNNUM]]_3 -; SI: BB[[FNNUM]]_3: -; SI: s_endpgm +; GCN: BB[[FNNUM]]_3: +; GCN: s_endpgm define void @cse_uniform_condition_different_blocks(i32 %cond, i32 addrspace(1)* %out) { bb: %tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0 @@ -434,6 +428,136 @@ bb9: ; preds = %bb8, %bb4 ret void } +; GCN-LABEL: {{^}}uniform_if_scc_i64_eq: +; VI-DAG: s_cmp_eq_u64 s{{\[[0-9]+:[0-9]+\]}}, 0 +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 + +; SI: v_cmp_eq_u64_e64 +; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] + +; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] + +; Fall-through to the else +; GCN: s_mov_b32 [[S_VAL]], 1 + +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] +define void @uniform_if_scc_i64_eq(i64 %cond, i32 addrspace(1)* %out) { +entry: + %cmp0 = icmp eq i64 %cond, 0 + br i1 %cmp0, label %if, label %else + +if: + br label %done + +else: + br label %done + +done: + %value = phi i32 [0, %if], [1, %else] + store i32 %value, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}uniform_if_scc_i64_ne: +; VI-DAG: s_cmp_lg_u64 s{{\[[0-9]+:[0-9]+\]}}, 0 +; GCN-DAG: s_mov_b32 [[S_VAL:s[0-9]+]], 0 + +; SI: v_cmp_ne_u64_e64 +; SI: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] + +; VI: s_cbranch_scc1 [[IF_LABEL:[0-9_A-Za-z]+]] + +; Fall-through to the else +; GCN: s_mov_b32 [[S_VAL]], 1 + +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL:v[0-9]+]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] +define void @uniform_if_scc_i64_ne(i64 %cond, i32 addrspace(1)* %out) { +entry: + %cmp0 = icmp ne i64 %cond, 0 + br i1 %cmp0, label %if, label %else + +if: + br label %done + +else: + br label %done + +done: + %value = phi i32 [0, %if], [1, %else] + store i32 %value, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}uniform_if_scc_i64_sgt: +; GCN: s_mov_b32 [[S_VAL:s[0-9]+]], 0 +; GCN: v_cmp_gt_i64_e64 +; GCN: s_cbranch_vccnz [[IF_LABEL:[0-9_A-Za-z]+]] + +; Fall-through to the else +; GCN: s_mov_b32 [[S_VAL]], 1 + +; GCN: [[IF_LABEL]]: +; GCN: v_mov_b32_e32 [[V_VAL]], [[S_VAL]] +; GCN: buffer_store_dword [[V_VAL]] +define void @uniform_if_scc_i64_sgt(i64 %cond, i32 addrspace(1)* %out) { +entry: + %cmp0 = icmp sgt i64 %cond, 0 + br i1 %cmp0, label %if, label %else + +if: + br label %done + +else: + br label %done + +done: + %value = phi i32 [0, %if], [1, %else] + store i32 %value, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}move_to_valu_i64_eq: +; GCN: v_cmp_eq_u64_e32 +define void @move_to_valu_i64_eq(i32 addrspace(1)* %out) { + %cond = load volatile i64, i64 addrspace(3)* undef + %cmp0 = icmp eq i64 %cond, 0 + br i1 %cmp0, label %if, label %else + +if: + br label %done + +else: + br label %done + +done: + %value = phi i32 [0, %if], [1, %else] + store i32 %value, i32 addrspace(1)* %out + ret void +} + +; GCN-LABEL: {{^}}move_to_valu_i64_ne: +; GCN: v_cmp_ne_u64_e32 +define void @move_to_valu_i64_ne(i32 addrspace(1)* %out) { + %cond = load volatile i64, i64 addrspace(3)* undef + %cmp0 = icmp ne i64 %cond, 0 + br i1 %cmp0, label %if, label %else + +if: + br label %done + +else: + br label %done + +done: + %value = phi i32 [0, %if], [1, %else] + store i32 %value, i32 addrspace(1)* %out + ret void +} + declare i32 @llvm.amdgcn.workitem.id.x() #0 -attributes #0 = { readnone } +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll b/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll index 9f2f0d67d245..2c3a09818860 100644 --- a/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll +++ b/test/CodeGen/AMDGPU/uniform-loop-inside-nonuniform.ll @@ -3,12 +3,15 @@ ; Test a simple uniform loop that lives inside non-uniform control flow. ; CHECK-LABEL: {{^}}test1: -; CHECK: v_cmp_ne_i32_e32 vcc, 0 +; CHECK: v_cmp_ne_u32_e32 vcc, 0 ; CHECK: s_and_saveexec_b64 +; CHECK-NEXT: s_xor_b64 +; CHECK-NEXT: ; mask branch + +; CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %loop_body.preheader ; CHECK: [[LOOP_BODY_LABEL:BB[0-9]+_[0-9]+]]: -; CHECK: s_and_b64 vcc, exec, vcc -; CHECK: s_cbranch_vccz [[LOOP_BODY_LABEL]] +; CHECK: s_cbranch_scc0 [[LOOP_BODY_LABEL]] ; CHECK: s_endpgm define amdgpu_ps void @test1(<8 x i32> inreg %rsrc, <2 x i32> %addr.base, i32 %y, i32 %p) { @@ -30,10 +33,11 @@ out: ret void } -;CHECK-LABEL: {{^}}test2: -;CHECK: s_and_saveexec_b64 -;CHECK: s_xor_b64 -;CHECK-NEXT: s_cbranch_execz +; CHECK-LABEL: {{^}}test2: +; CHECK: s_and_saveexec_b64 +; CHECK-NEXT: s_xor_b64 +; CHECK-NEXT: ; mask branch +; CHECK-NEXT: s_cbranch_execz define void @test2(i32 addrspace(1)* %out, i32 %a, i32 %b) { main_body: %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 diff --git a/test/CodeGen/AMDGPU/unify-metadata.ll b/test/CodeGen/AMDGPU/unify-metadata.ll new file mode 100644 index 000000000000..9549b08ffee1 --- /dev/null +++ b/test/CodeGen/AMDGPU/unify-metadata.ll @@ -0,0 +1,31 @@ +; RUN: opt -mtriple=amdgcn--amdhsa -amdgpu-unify-metadata -S < %s | FileCheck -check-prefix=ALL %s + +; This test check that we have a singe metadata value after linking several +; modules for records such as opencl.ocl.version, llvm.ident and similar. + +; ALL-DAG: !opencl.ocl.version = !{![[OCL_VER:[0-9]+]]} +; ALL-DAG: !llvm.ident = !{![[LLVM_IDENT_0:[0-9]+]], ![[LLVM_IDENT_1:[0-9]+]]} +; ALL-DAG: !opencl.used.extensions = !{![[USED_EXT_0:[0-9]+]], ![[USED_EXT_1:[0-9]+]], ![[USED_EXT_2:[0-9]+]]} + +; ALL-DAG: ![[OCL_VER]] = !{i32 1, i32 2} +; ALL-DAG: ![[LLVM_IDENT_0]] = !{!"clang version 4.0"} +; ALL-DAG: ![[LLVM_IDENT_1]] = !{!"clang version 4.0 (rLXXXXXX)"} +; ALL-DAG: ![[USED_EXT_0]] = !{!"cl_images"} +; ALL-DAG: ![[USED_EXT_1]] = !{!"cl_khr_fp16"} +; ALL-DAG: ![[USED_EXT_2]] = !{!"cl_doubles"} + +define void @test() { + ret void +} + +!opencl.ocl.version = !{!1, !0, !0, !0} +!llvm.ident = !{!2, !2, !2, !2, !6} +!opencl.used.extensions = !{!3, !3, !4, !5} + +!0 = !{i32 2, i32 0} +!1 = !{i32 1, i32 2} +!2 = !{!"clang version 4.0"} +!3 = !{!"cl_images", !"cl_khr_fp16"} +!4 = !{!"cl_images", !"cl_doubles"} +!5 = !{} +!6 = !{!"clang version 4.0 (rLXXXXXX)"} diff --git a/test/CodeGen/AMDGPU/unigine-liveness-crash.ll b/test/CodeGen/AMDGPU/unigine-liveness-crash.ll new file mode 100644 index 000000000000..732790ceb335 --- /dev/null +++ b/test/CodeGen/AMDGPU/unigine-liveness-crash.ll @@ -0,0 +1,115 @@ +; RUN: llc -march=amdgcn < %s | FileCheck %s +; REQUIRES: asserts +; +; This test used to crash with the following assertion: +; llc: include/llvm/ADT/IntervalMap.h:632: unsigned int llvm::IntervalMapImpl::LeafNode<llvm::SlotIndex, llvm::LiveInterval *, 8, llvm::IntervalMapInfo<llvm::SlotIndex> >::insertFrom(unsigned int &, unsigned int, KeyT, KeyT, ValT) [KeyT = llvm::SlotIndex, ValT = llvm::LiveInterval *, N = 8, Traits = llvm::IntervalMapInfo<llvm::SlotIndex>]: Assertion `(i == Size || Traits::stopLess(b, start(i))) && "Overlapping insert"' failed. +; +; This was related to incorrectly calculating subregister live ranges +; (i.e. live interval subranges): subregister defs are not uses for that +; purpose. +; +; Check for a valid output. +; CHECK: image_sample_c + +target triple = "amdgcn--" + +@ddxy_lds = external addrspace(3) global [64 x i32] + +define amdgpu_ps <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> @main([17 x <16 x i8>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg, [16 x <16 x i8>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg1, [32 x <8 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg2, [16 x <8 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg3, [16 x <4 x i32>] addrspace(2)* byval dereferenceable(18446744073709551615) %arg4, float inreg %arg5, i32 inreg %arg6, <2 x i32> %arg7, <2 x i32> %arg8, <2 x i32> %arg9, <3 x i32> %arg10, <2 x i32> %arg11, <2 x i32> %arg12, <2 x i32> %arg13, float %arg14, float %arg15, float %arg16, float %arg17, float %arg18, i32 %arg19, i32 %arg20, float %arg21, i32 %arg22) #0 { +main_body: + %tmp = call float @llvm.SI.fs.interp(i32 3, i32 4, i32 %arg6, <2 x i32> %arg8) + %tmp23 = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp24 = extractelement <4 x float> %tmp23, i32 3 + %tmp25 = fmul float %tmp24, undef + %tmp26 = fmul float undef, %tmp + %tmp27 = fadd float %tmp26, undef + %tmp28 = bitcast float %tmp27 to i32 + %tmp29 = insertelement <4 x i32> undef, i32 %tmp28, i32 0 + %tmp30 = insertelement <4 x i32> %tmp29, i32 0, i32 1 + %tmp31 = insertelement <4 x i32> %tmp30, i32 undef, i32 2 + %tmp32 = call <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32> %tmp31, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp33 = extractelement <4 x float> %tmp32, i32 0 + %tmp34 = fadd float undef, %tmp33 + %tmp35 = fadd float %tmp34, undef + %tmp36 = fadd float %tmp35, undef + %tmp37 = fadd float %tmp36, undef + %tmp38 = fadd float %tmp37, undef + %tmp39 = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> undef, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp40 = extractelement <4 x float> %tmp39, i32 0 + %tmp41 = extractelement <4 x float> %tmp39, i32 1 + %tmp42 = extractelement <4 x float> %tmp39, i32 2 + %tmp43 = extractelement <4 x float> %tmp39, i32 3 + %tmp44 = fmul float %tmp40, undef + %tmp45 = fmul float %tmp41, undef + %tmp46 = fmul float %tmp42, undef + %tmp47 = fmul float %tmp43, undef + %tmp48 = fadd float undef, %tmp44 + %tmp49 = fadd float undef, %tmp45 + %tmp50 = bitcast float %tmp27 to i32 + %tmp51 = bitcast float %tmp48 to i32 + %tmp52 = bitcast float %tmp49 to i32 + %tmp53 = insertelement <4 x i32> undef, i32 %tmp50, i32 0 + %tmp54 = insertelement <4 x i32> %tmp53, i32 %tmp51, i32 1 + %tmp55 = insertelement <4 x i32> %tmp54, i32 %tmp52, i32 2 + %tmp56 = call <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32> %tmp55, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp57 = extractelement <4 x float> %tmp56, i32 0 + %tmp58 = fadd float %tmp38, %tmp57 + %tmp59 = fadd float undef, %tmp46 + %tmp60 = fadd float undef, %tmp47 + %tmp61 = bitcast float %tmp59 to i32 + %tmp62 = bitcast float %tmp60 to i32 + %tmp63 = insertelement <4 x i32> undef, i32 %tmp61, i32 1 + %tmp64 = insertelement <4 x i32> %tmp63, i32 %tmp62, i32 2 + %tmp65 = call <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32> %tmp64, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp66 = extractelement <4 x float> %tmp65, i32 0 + %tmp67 = fadd float %tmp58, %tmp66 + %tmp68 = fmul float %tmp67, 1.250000e-01 + %tmp69 = fmul float %tmp68, undef + %tmp70 = fcmp une float %tmp69, 0.000000e+00 + br i1 %tmp70, label %IF26, label %ENDIF25 + +IF26: ; preds = %main_body + %tmp71 = bitcast float %tmp27 to i32 + %tmp72 = insertelement <4 x i32> undef, i32 %tmp71, i32 0 + br label %LOOP + +ENDIF25: ; preds = %IF29, %main_body + %.4 = phi float [ %tmp84, %IF29 ], [ %tmp68, %main_body ] + %tmp73 = fadd float %.4, undef + %tmp74 = call float @llvm.AMDGPU.clamp.(float %tmp73, float 0.000000e+00, float 1.000000e+00) + %tmp75 = fmul float undef, %tmp74 + %tmp76 = fmul float %tmp75, undef + %tmp77 = fadd float %tmp76, undef + %tmp78 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> undef, float %tmp77, 11 + %tmp79 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp78, float undef, 12 + %tmp80 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp79, float undef, 13 + %tmp81 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp80, float %tmp25, 14 + %tmp82 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp81, float undef, 15 + %tmp83 = insertvalue <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp82, float %arg21, 24 + ret <{ i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, float, float, float, float, float, float, float, float, float, float, float, float, float, float }> %tmp83 + +LOOP: ; preds = %ENDIF28, %IF26 + %.5 = phi float [ undef, %IF26 ], [ %tmp89, %ENDIF28 ] + br i1 false, label %IF29, label %ENDIF28 + +IF29: ; preds = %LOOP + %tmp84 = fmul float %.5, 3.125000e-02 + br label %ENDIF25 + +ENDIF28: ; preds = %LOOP + %tmp85 = insertelement <4 x i32> %tmp72, i32 undef, i32 1 + %tmp86 = insertelement <4 x i32> %tmp85, i32 undef, i32 2 + %tmp87 = call <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32> %tmp86, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tmp88 = extractelement <4 x float> %tmp87, i32 0 + %tmp89 = fadd float undef, %tmp88 + br label %LOOP +} + +declare float @llvm.AMDGPU.clamp.(float, float, float) #1 +declare float @llvm.SI.fs.interp(i32, i32, i32, <2 x i32>) #1 +declare <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #1 +declare <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #1 +declare <4 x float> @llvm.SI.image.sample.c.v4i32(<4 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #1 + +attributes #0 = { "InitialPSInputAddr"="36983" "target-cpu"="tonga" } +attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll b/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll index 0435ed4d5525..87fcfbade14a 100644 --- a/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll +++ b/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll @@ -44,15 +44,16 @@ define void @test_sgpr_use_twice_ternary_op_a_a_b(float addrspace(1)* %out, floa ; GCN-LABEL: {{^}}test_use_s_v_s: ; GCN-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} ; GCN-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, {{0xc|0x30}} - -; GCN: buffer_load_dword [[VA0:v[0-9]+]] -; GCN-NOT: v_mov_b32 -; GCN: buffer_load_dword [[VA1:v[0-9]+]] +; SI: buffer_load_dword [[VA0:v[0-9]+]] +; SI: buffer_load_dword [[VA1:v[0-9]+]] ; GCN-NOT: v_mov_b32 ; GCN: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]] ; GCN-NOT: v_mov_b32 +; VI: buffer_load_dword [[VA0:v[0-9]+]] +; VI: buffer_load_dword [[VA1:v[0-9]+]] + ; GCN-DAG: v_fma_f32 [[RESULT0:v[0-9]+]], [[VA0]], [[SA]], [[VB]] ; GCN-DAG: v_fma_f32 [[RESULT1:v[0-9]+]], [[VA1]], [[SA]], [[VB]] ; GCN: buffer_store_dword [[RESULT0]] diff --git a/test/CodeGen/AMDGPU/v1i64-kernel-arg.ll b/test/CodeGen/AMDGPU/v1i64-kernel-arg.ll index 31755125c03b..a48e7acd4cf3 100644 --- a/test/CodeGen/AMDGPU/v1i64-kernel-arg.ll +++ b/test/CodeGen/AMDGPU/v1i64-kernel-arg.ll @@ -1,5 +1,3 @@ -; REQUIRES: asserts -; XFAIL: * ; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck %s ; CHECK-LABEL: {{^}}kernel_arg_i64: diff --git a/test/CodeGen/AMDGPU/v_cndmask.ll b/test/CodeGen/AMDGPU/v_cndmask.ll index ca6bff4f6fc8..1c61671f9cd3 100644 --- a/test/CodeGen/AMDGPU/v_cndmask.ll +++ b/test/CodeGen/AMDGPU/v_cndmask.ll @@ -1,17 +1,18 @@ -; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s -; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s declare i32 @llvm.amdgcn.workitem.id.x() #1 -; SI-LABEL: {{^}}v_cnd_nan_nosgpr: -; SI: v_cndmask_b32_e64 v{{[0-9]}}, v{{[0-9]}}, -1, s{{\[[0-9]+:[0-9]+\]}} -; SI-DAG: v{{[0-9]}} +; GCN-LABEL: {{^}}v_cnd_nan_nosgpr: +; GCN: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0 +; GCN: v_cndmask_b32_e32 v{{[0-9]}}, -1, v{{[0-9]+}}, vcc +; GCN-DAG: v{{[0-9]}} ; All nan values are converted to 0xffffffff -; SI: s_endpgm +; GCN: s_endpgm define void @v_cnd_nan_nosgpr(float addrspace(1)* %out, i32 %c, float addrspace(1)* %fptr) #0 { %idx = call i32 @llvm.amdgcn.workitem.id.x() #1 %f.gep = getelementptr float, float addrspace(1)* %fptr, i32 %idx - %f = load float, float addrspace(1)* %fptr + %f = load float, float addrspace(1)* %f.gep %setcc = icmp ne i32 %c, 0 %select = select i1 %setcc, float 0xFFFFFFFFE0000000, float %f store float %select, float addrspace(1)* %out @@ -23,11 +24,12 @@ define void @v_cnd_nan_nosgpr(float addrspace(1)* %out, i32 %c, float addrspace( ; single constant bus SGPR usage is the last operand, and it should ; never be moved. -; SI-LABEL: {{^}}v_cnd_nan: -; SI: v_cndmask_b32_e64 v{{[0-9]}}, v{{[0-9]}}, -1, s{{\[[0-9]+:[0-9]+\]}} -; SI-DAG: v{{[0-9]}} +; GCN-LABEL: {{^}}v_cnd_nan: +; GCN: v_cmp_eq_u32_e64 vcc, s{{[0-9]+}}, 0 +; GCN: v_cndmask_b32_e32 v{{[0-9]}}, -1, v{{[0-9]}}, vcc +; GCN-DAG: v{{[0-9]}} ; All nan values are converted to 0xffffffff -; SI: s_endpgm +; GCN: s_endpgm define void @v_cnd_nan(float addrspace(1)* %out, i32 %c, float %f) #0 { %setcc = icmp ne i32 %c, 0 %select = select i1 %setcc, float 0xFFFFFFFFE0000000, float %f @@ -35,5 +37,370 @@ define void @v_cnd_nan(float addrspace(1)* %out, i32 %c, float %f) #0 { ret void } +; Test different compare and select operand types for optimal code +; shrinking. +; (select (cmp (sgprX, constant)), constant, sgprZ) + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k1_sgprZ_f32: +; GCN: s_load_dword [[X:s[0-9]+]] +; GCN: s_load_dword [[Z:s[0-9]+]] +; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[VZ]], vcc +define void @fcmp_sgprX_k0_select_k1_sgprZ_f32(float addrspace(1)* %out, float %x, float %z) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 1.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k1_sgprX_f32: +; GCN: s_load_dword [[X:s[0-9]+]] +; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[VZ]], vcc +define void @fcmp_sgprX_k0_select_k1_sgprX_f32(float addrspace(1)* %out, float %x) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 1.0, float %x + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k0_sgprZ_f32: +; GCN: s_load_dword [[X:s[0-9]+]] +; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[VZ]], vcc +define void @fcmp_sgprX_k0_select_k0_sgprZ_f32(float addrspace(1)* %out, float %x, float %z) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 0.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k0_sgprX_f32: +; GCN: s_load_dword [[X:s[0-9]+]] +; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[VZ]], vcc +define void @fcmp_sgprX_k0_select_k0_sgprX_f32(float addrspace(1)* %out, float %x) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 0.0, float %x + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k0_vgprZ_f32: +; GCN-DAG: s_load_dword [[X:s[0-9]+]] +; GCN-DAG: {{buffer|flat}}_load_dword [[Z:v[0-9]+]] +; GCN-DAG: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 0, [[Z]], vcc +define void @fcmp_sgprX_k0_select_k0_vgprZ_f32(float addrspace(1)* %out, float %x, float addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %z.gep = getelementptr inbounds float, float addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %z = load float, float addrspace(1)* %z.gep + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 0.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_sgprX_k0_select_k1_vgprZ_f32: +; GCN-DAG: {{buffer|flat}}_load_dword [[Z:v[0-9]+]] +; GCN-DAG: s_load_dword [[X:s[0-9]+]] +; GCN: v_cmp_nlg_f32_e64 vcc, [[X]], 0 +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[Z]], vcc +define void @fcmp_sgprX_k0_select_k1_vgprZ_f32(float addrspace(1)* %out, float %x, float addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %z.gep = getelementptr inbounds float, float addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %z = load float, float addrspace(1)* %z.gep + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, float 1.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_vgprX_k0_select_k1_sgprZ_f32: +; GCN-DAG: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN-DAG: s_load_dword [[Z:s[0-9]+]] +; GCN-DAG: v_cmp_ngt_f32_e32 vcc, 0, [[X]] +; GCN-DAG: v_mov_b32_e32 [[VZ:v[0-9]+]], [[Z]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[VZ]], vcc +define void @fcmp_vgprX_k0_select_k1_sgprZ_f32(float addrspace(1)* %out, float addrspace(1)* %x.ptr, float %z) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %x = load float, float addrspace(1)* %x.gep + %setcc = fcmp olt float %x, 0.0 + %select = select i1 %setcc, float 1.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_vgprX_k0_select_k1_vgprZ_f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[Z:v[0-9]+]] +; GCN: v_cmp_le_f32_e32 vcc, 0, [[X]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[Z]], vcc +define void @fcmp_vgprX_k0_select_k1_vgprZ_f32(float addrspace(1)* %out, float addrspace(1)* %x.ptr, float addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds float, float addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile float, float addrspace(1)* %z.gep + %setcc = fcmp ult float %x, 0.0 + %select = select i1 %setcc, float 1.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}icmp_vgprX_k0_select_k1_vgprZ_i32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[Z:v[0-9]+]] +; GCN: v_cmp_lt_i32_e32 vcc, -1, [[X]] +; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, 2, [[Z]], vcc +define void @icmp_vgprX_k0_select_k1_vgprZ_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %x.ptr, i32 addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds i32, i32 addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds i32, i32 addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 %tid.ext + %x = load volatile i32, i32 addrspace(1)* %x.gep + %z = load volatile i32, i32 addrspace(1)* %z.gep + %setcc = icmp slt i32 %x, 0 + %select = select i1 %setcc, i32 2, i32 %z + store i32 %select, i32 addrspace(1)* %out.gep + ret void +} + +; FIXME: Why does VI make the wrong regalloc choice? +; GCN-LABEL: {{^}}icmp_vgprX_k0_select_k1_vgprZ_i64: +; GCN: {{buffer|flat}}_load_dwordx2 v{{\[}}[[X_LO:[0-9]+]]:[[X_HI:[0-9]+]]{{\]}} +; GCN-DAG: {{buffer|flat}}_load_dwordx2 v{{\[}}[[Z_LO:[0-9]+]]:[[Z_HI:[0-9]+]]{{\]}} +; SI-DAG: v_cmp_lt_i64_e32 vcc, -1, v{{\[}}[[X_LO]]:[[X_HI]]{{\]}} +; SI-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v[[Z_HI]], vcc +; SI-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 2, v[[Z_LO]], vcc + +; VI-DAG: v_cmp_lt_i64_e64 s{{\[[0-9]+:[0-9]+\]}}, -1, v{{\[}}[[X_LO]]:[[X_HI]]{{\]}} +; VI-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 0, v[[Z_HI]], s +; VI-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 2, v[[Z_LO]], s +define void @icmp_vgprX_k0_select_k1_vgprZ_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %x.ptr, i64 addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds i64, i64 addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds i64, i64 addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i64, i64 addrspace(1)* %out, i64 %tid.ext + %x = load volatile i64, i64 addrspace(1)* %x.gep + %z = load volatile i64, i64 addrspace(1)* %z.gep + %setcc = icmp slt i64 %x, 0 + %select = select i1 %setcc, i64 2, i64 %z + store i64 %select, i64 addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_vgprX_k0_select_vgprZ_k1_v4f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dwordx4 + +; GCN: v_cmp_nge_f32_e32 vcc, 4.0, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 2.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, -0.5, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 4.0, v{{[0-9]+}}, vcc +define void @fcmp_vgprX_k0_select_vgprZ_k1_v4f32(<4 x float> addrspace(1)* %out, float addrspace(1)* %x.ptr, <4 x float> addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile <4 x float>, <4 x float> addrspace(1)* %z.gep + %setcc = fcmp ugt float %x, 4.0 + %select = select i1 %setcc, <4 x float> %z, <4 x float> <float 1.0, float 2.0, float -0.5, float 4.0> + store <4 x float> %select, <4 x float> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}fcmp_vgprX_k0_select_k1_vgprZ_v4f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dwordx4 + +; GCN: v_cmp_ge_f32_e32 vcc, 4.0, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 2.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, -0.5, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 4.0, v{{[0-9]+}}, vcc +define void @fcmp_vgprX_k0_select_k1_vgprZ_v4f32(<4 x float> addrspace(1)* %out, float addrspace(1)* %x.ptr, <4 x float> addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile <4 x float>, <4 x float> addrspace(1)* %z.gep + %setcc = fcmp ugt float %x, 4.0 + %select = select i1 %setcc, <4 x float> <float 1.0, float 2.0, float -0.5, float 4.0>, <4 x float> %z + store <4 x float> %select, <4 x float> addrspace(1)* %out.gep + ret void +} + +; This must be swapped as a vector type before the condition has +; multiple uses. + +; GCN-LABEL: {{^}}fcmp_k0_vgprX_select_k1_vgprZ_v4f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dwordx4 + +; GCN: v_cmp_le_f32_e32 vcc, 4.0, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 2.0, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, -0.5, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 4.0, v{{[0-9]+}}, vcc +define void @fcmp_k0_vgprX_select_k1_vgprZ_v4f32(<4 x float> addrspace(1)* %out, float addrspace(1)* %x.ptr, <4 x float> addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile <4 x float>, <4 x float> addrspace(1)* %z.gep + %setcc = fcmp ugt float 4.0, %x + %select = select i1 %setcc, <4 x float> <float 1.0, float 2.0, float -0.5, float 4.0>, <4 x float> %z + store <4 x float> %select, <4 x float> addrspace(1)* %out.gep + ret void +} + +; GCN-LABEL: {{^}}icmp_vgprX_k0_select_k1_vgprZ_i1: +; GCN: load_dword +; GCN: load_ubyte +; GCN-DAG: v_cmp_gt_i32_e64 s{{\[[0-9]+:[0-9]+\]}}, 0, v +; DCN-DAG: v_and_b32_e32 v{{[0-9]+}}, 1, +; GCN-DAG: v_cmp_eq_u32_e32 vcc, 1, v +; GCN-DAG: s_or_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, vcc +; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, s +; GCN: store_byte +define void @icmp_vgprX_k0_select_k1_vgprZ_i1(i1 addrspace(1)* %out, i32 addrspace(1)* %x.ptr, i1 addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds i32, i32 addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds i1, i1 addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i1, i1 addrspace(1)* %out, i64 %tid.ext + %x = load volatile i32, i32 addrspace(1)* %x.gep + %z = load volatile i1, i1 addrspace(1)* %z.gep + %setcc = icmp slt i32 %x, 0 + %select = select i1 %setcc, i1 true, i1 %z + store i1 %select, i1 addrspace(1)* %out.gep + ret void +} + +; Different types compared vs. selected +; GCN-LABEL: {{^}}fcmp_vgprX_k0_selectf64_k1_vgprZ_f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dwordx2 + +; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0x3ff00000 +; GCN: v_cmp_le_f32_e32 vcc, 0, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, [[K]], v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}, vcc +define void @fcmp_vgprX_k0_selectf64_k1_vgprZ_f32(double addrspace(1)* %out, float addrspace(1)* %x.ptr, double addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds double, double addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile double, double addrspace(1)* %z.gep + %setcc = fcmp ult float %x, 0.0 + %select = select i1 %setcc, double 1.0, double %z + store double %select, double addrspace(1)* %out.gep + ret void +} + +; Different types compared vs. selected +; GCN-LABEL: {{^}}fcmp_vgprX_k0_selecti64_k1_vgprZ_f32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dwordx2 + +; GCN: v_cmp_nlg_f32_e32 vcc, 0, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 3, v{{[0-9]+}}, vcc +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 0, v{{[0-9]+}}, vcc +define void @fcmp_vgprX_k0_selecti64_k1_vgprZ_f32(i64 addrspace(1)* %out, float addrspace(1)* %x.ptr, i64 addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds i64, i64 addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds i64, i64 addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile i64, i64 addrspace(1)* %z.gep + %setcc = fcmp one float %x, 0.0 + %select = select i1 %setcc, i64 3, i64 %z + store i64 %select, i64 addrspace(1)* %out.gep + ret void +} + +; Different types compared vs. selected +; GCN-LABEL: {{^}}icmp_vgprX_k0_selectf32_k1_vgprZ_i32: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] +; GCN: {{buffer|flat}}_load_dword [[Z:v[0-9]+]] + +; GCN: v_cmp_gt_u32_e32 vcc, 2, [[X]] +; GCN-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 4.0, [[Z]], vcc +define void @icmp_vgprX_k0_selectf32_k1_vgprZ_i32(float addrspace(1)* %out, i32 addrspace(1)* %x.ptr, float addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds i32, i32 addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds float, float addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %x = load volatile i32, i32 addrspace(1)* %x.gep + %z = load volatile float, float addrspace(1)* %z.gep + %setcc = icmp ugt i32 %x, 1 + %select = select i1 %setcc, float 4.0, float %z + store float %select, float addrspace(1)* %out.gep + ret void +} + +; FIXME: Should be able to handle multiple uses + +; GCN-LABEL: {{^}}fcmp_k0_vgprX_select_k1_vgprZ_f32_cond_use_x2: +; GCN: {{buffer|flat}}_load_dword [[X:v[0-9]+]] + +; GCN: v_cmp_nle_f32_e32 vcc, 4.0, [[X]] +; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -1.0, vcc +; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -2.0, vcc +define void @fcmp_k0_vgprX_select_k1_vgprZ_f32_cond_use_x2(float addrspace(1)* %out, float addrspace(1)* %x.ptr, float addrspace(1)* %z.ptr) #0 { + %tid = call i32 @llvm.amdgcn.workitem.id.x() #1 + %tid.ext = sext i32 %tid to i64 + %x.gep = getelementptr inbounds float, float addrspace(1)* %x.ptr, i64 %tid.ext + %z.gep = getelementptr inbounds float, float addrspace(1)* %z.ptr, i64 %tid.ext + %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i64 %tid.ext + %x = load volatile float, float addrspace(1)* %x.gep + %z = load volatile float, float addrspace(1)* %z.gep + %setcc = fcmp ugt float 4.0, %x + %select0 = select i1 %setcc, float -1.0, float %z + %select1 = select i1 %setcc, float -2.0, float %z + store volatile float %select0, float addrspace(1)* %out.gep + store volatile float %select1, float addrspace(1)* %out.gep + ret void +} + attributes #0 = { nounwind } attributes #1 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/v_cvt_pk_u8_f32.ll b/test/CodeGen/AMDGPU/v_cvt_pk_u8_f32.ll new file mode 100644 index 000000000000..9246ce38dbed --- /dev/null +++ b/test/CodeGen/AMDGPU/v_cvt_pk_u8_f32.ll @@ -0,0 +1,60 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s + +declare i32 @llvm.amdgcn.cvt.pk.u8.f32(float, i32, i32) #0 + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_idx_0: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 0, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_idx_0(i32 addrspace(1)* %out, float %src, i32 %reg) { + %result = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 0, i32 %reg) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_idx_1: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 1, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_idx_1(i32 addrspace(1)* %out, float %src, i32 %reg) { + %result = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 1, i32 %reg) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_idx_2: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 2, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_idx_2(i32 addrspace(1)* %out, float %src, i32 %reg) { + %result = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 2, i32 %reg) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_idx_3: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 3, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_idx_3(i32 addrspace(1)* %out, float %src, i32 %reg) { + %result = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 3, i32 %reg) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_combine: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 0, v{{[0-9]+}} +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 1, v{{[0-9]+}} +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 2, v{{[0-9]+}} +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, 3, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_combine(i32 addrspace(1)* %out, float %src, i32 %reg) { + %result0 = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 0, i32 %reg) #0 + %result1 = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 1, i32 %result0) #0 + %result2 = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 2, i32 %result1) #0 + %result3 = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 3, i32 %result2) #0 + store i32 %result3, i32 addrspace(1)* %out, align 4 + ret void +} + +; GCN-LABEL: {{^}}v_cvt_pk_u8_f32_idx: +; GCN: v_cvt_pk_u8_f32 v{{[0-9]+}}, s{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +define void @v_cvt_pk_u8_f32_idx(i32 addrspace(1)* %out, float %src, i32 %idx, i32 %reg) { + %result = call i32 @llvm.amdgcn.cvt.pk.u8.f32(float %src, i32 %idx, i32 %reg) #0 + store i32 %result, i32 addrspace(1)* %out, align 4 + ret void +} + +attributes #0 = { nounwind readnone } diff --git a/test/CodeGen/AMDGPU/v_mac_f16.ll b/test/CodeGen/AMDGPU/v_mac_f16.ll new file mode 100644 index 000000000000..ecd5b01545d6 --- /dev/null +++ b/test/CodeGen/AMDGPU/v_mac_f16.ll @@ -0,0 +1,608 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}mac_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[C_F16:[0-9]+]] +; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32:[0-9]+]], v[[C_F16]] +; SI: v_mac_f32_e32 v[[C_F32]], v[[B_F32]], v[[A_F32]] +; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[C_F32]] +; SI: buffer_store_short v[[R_F16]] +; VI: v_mac_f16_e32 v[[C_F16]], v[[B_F16]], v[[A_F16]] +; VI: buffer_store_short v[[C_F16]] +; GCN: s_endpgm +define void @mac_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %t.val = fmul half %a.val, %b.val + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_same_add +; SI: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]] +; SI: v_mac_f32_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]] +; VI: v_mac_f16_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_same_add( + half addrspace(1)* %r0, + half addrspace(1)* %r1, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c, + half addrspace(1)* %d, + half addrspace(1)* %e) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + %d.val = load half, half addrspace(1)* %d + %e.val = load half, half addrspace(1)* %e + + %t0.val = fmul half %a.val, %b.val + %r0.val = fadd half %t0.val, %c.val + + %t1.val = fmul half %d.val, %e.val + %r1.val = fadd half %t1.val, %c.val + + store half %r0.val, half addrspace(1)* %r0 + store half %r1.val, half addrspace(1)* %r1 + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_a +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_neg_a( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %a.neg = fsub half -0.0, %a.val + %t.val = fmul half %a.neg, %b.val + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_b +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_neg_b( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %b.neg = fsub half -0.0, %b.val + %t.val = fmul half %a.val, %b.neg + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_c +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_neg_c( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %c.neg = fsub half -0.0, %c.val + %t.val = fmul half %a.val, %b.val + %r.val = fadd half %t.val, %c.neg + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_a_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A]] +; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A]] +; GCN: s_endpgm +define void @mac_f16_neg_a_safe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %a.neg = fsub half 0.0, %a.val + %t.val = fmul half %a.neg, %b.val + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_b_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v{{[0-9]+}}, v[[NEG_A]], v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v[[NEG_A]], v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_neg_b_safe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %b.neg = fsub half 0.0, %b.val + %t.val = fmul half %a.val, %b.neg + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_c_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v[[NEG_A]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v[[NEG_A]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_f16_neg_c_safe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #0 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %c.neg = fsub half 0.0, %c.val + %t.val = fmul half %a.val, %b.val + %r.val = fadd half %t.val, %c.neg + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_a_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}} +; GCN: s_endpgm +define void @mac_f16_neg_a_unsafe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #1 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %a.neg = fsub half 0.0, %a.val + %t.val = fmul half %a.neg, %b.val + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_b_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]}} +; GCN: s_endpgm +define void @mac_f16_neg_b_unsafe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #1 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %b.neg = fsub half 0.0, %b.val + %t.val = fmul half %a.val, %b.neg + %r.val = fadd half %t.val, %c.val + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_f16_neg_c_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]}} +; GCN: s_endpgm +define void @mac_f16_neg_c_unsafe_fp_math( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) #1 { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %c.neg = fsub half 0.0, %c.val + %t.val = fmul half %a.val, %b.val + %r.val = fadd half %t.val, %c.neg + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16 +; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]] +; GCN: buffer_load_dword v[[C_V2_F16:[0-9]+]] +; GCN: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]] +; GCN: v_lshrrev_b32_e32 v[[C_F16_1:[0-9]+]], 16, v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_0:[0-9]+]], v[[C_V2_F16]] +; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]] +; SI: v_cvt_f32_f16_e32 v[[C_F32_1:[0-9]+]], v[[C_F16_1]] +; SI: v_mac_f32_e32 v[[C_F32_0]], v[[B_F32_0]], v[[A_F32_0]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[C_F32_0]] +; SI: v_mac_f32_e32 v[[C_F32_1]], v[[B_F32_1]], v[[A_F32_1]] +; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[C_F32_1]] +; SI: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[R_F16_0]] +; SI: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]] +; VI: v_mac_f16_e32 v[[C_V2_F16]], v[[B_V2_F16]], v[[A_V2_F16]] +; VI: v_mac_f16_e32 v[[C_F16_1]], v[[B_F16_1]], v[[A_F16_1]] +; VI: v_and_b32_e32 v[[R_F16_LO:[0-9]+]], 0xffff, v[[C_V2_F16]] +; VI: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[C_F16_1]] +; GCN: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_HI]], v[[R_F16_LO]] +; GCN: buffer_store_dword v[[R_V2_F16]] +; GCN: s_endpgm +define void @mac_v2f16( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %t.val = fmul <2 x half> %a.val, %b.val + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_same_add +; SI: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD0:v[0-9]+]] +; SI: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD1:v[0-9]+]] +; SI: v_mac_f32_e32 [[ADD0]], v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_mac_f32_e32 [[ADD1]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD0:v[0-9]+]] +; VI: v_mad_f16 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD1:v[0-9]+]] +; VI: v_mac_f16_e32 [[ADD0]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mac_f16_e32 [[ADD1]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_same_add( + <2 x half> addrspace(1)* %r0, + <2 x half> addrspace(1)* %r1, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c, + <2 x half> addrspace(1)* %d, + <2 x half> addrspace(1)* %e) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + %d.val = load <2 x half>, <2 x half> addrspace(1)* %d + %e.val = load <2 x half>, <2 x half> addrspace(1)* %e + + %t0.val = fmul <2 x half> %a.val, %b.val + %r0.val = fadd <2 x half> %t0.val, %c.val + + %t1.val = fmul <2 x half> %d.val, %e.val + %r1.val = fadd <2 x half> %t1.val, %c.val + + store <2 x half> %r0.val, <2 x half> addrspace(1)* %r0 + store <2 x half> %r1.val, <2 x half> addrspace(1)* %r1 + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_a +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_neg_a( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %a.neg = fsub <2 x half> <half -0.0, half -0.0>, %a.val + %t.val = fmul <2 x half> %a.neg, %b.val + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_b +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_neg_b( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %b.neg = fsub <2 x half> <half -0.0, half -0.0>, %b.val + %t.val = fmul <2 x half> %a.val, %b.neg + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_c +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_neg_c( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %c.neg = fsub <2 x half> <half -0.0, half -0.0>, %c.val + %t.val = fmul <2 x half> %a.val, %b.val + %r.val = fadd <2 x half> %t.val, %c.neg + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_a_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A0:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_subrev_f32_e32 v[[NEG_A1:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A0]] +; SI: v_mac_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A1]] +; VI: v_sub_f16_e32 v[[NEG_A0:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A0]] +; VI: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v[[NEG_A1]] +; GCN: s_endpgm +define void @mac_v2f16_neg_a_safe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %a.neg = fsub <2 x half> <half 0.0, half 0.0>, %a.val + %t.val = fmul <2 x half> %a.neg, %b.val + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_b_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A0:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_subrev_f32_e32 v[[NEG_A1:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v{{[0-9]+}}, v[[NEG_A0]], v{{[0-9]+}} +; SI: v_mac_f32_e32 v{{[0-9]+}}, v[[NEG_A1]], v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A0:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v[[NEG_A0]], v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v[[NEG_A1]], v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_neg_b_safe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %b.neg = fsub <2 x half> <half 0.0, half 0.0>, %b.val + %t.val = fmul <2 x half> %a.val, %b.neg + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_c_safe_fp_math +; SI: v_cvt_f32_f16_e32 v[[ZERO:[0-9]+]], 0{{$}} +; SI: v_subrev_f32_e32 v[[NEG_A0:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_subrev_f32_e32 v[[NEG_A1:[0-9]+]], v{{[0-9]+}}, v[[ZERO]] +; SI: v_mac_f32_e32 v[[NEG_A0]], v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_mac_f32_e32 v[[NEG_A1]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A0:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_sub_f16_e32 v[[NEG_A1:[0-9]+]], 0, v{{[0-9]+}} +; VI: v_mac_f16_e32 v[[NEG_A0]], v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mac_f16_e32 v[[NEG_A1]], v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @mac_v2f16_neg_c_safe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #0 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %c.neg = fsub <2 x half> <half 0.0, half 0.0>, %c.val + %t.val = fmul <2 x half> %a.val, %b.val + %r.val = fadd <2 x half> %t.val, %c.neg + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_a_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; GCN: s_endpgm +define void @mac_v2f16_neg_a_unsafe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #1 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %a.neg = fsub <2 x half> <half 0.0, half 0.0>, %a.val + %t.val = fmul <2 x half> %a.neg, %b.val + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_b_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; SI: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; VI: v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}} +; GCN: s_endpgm +define void @mac_v2f16_neg_b_unsafe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #1 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %b.neg = fsub <2 x half> <half 0.0, half 0.0>, %b.val + %t.val = fmul <2 x half> %a.val, %b.neg + %r.val = fadd <2 x half> %t.val, %c.val + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}mac_v2f16_neg_c_unsafe_fp_math +; SI-NOT: v_mac_f32 +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}} +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}} +; VI-NOT: v_mac_f16 +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}} +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}} +; GCN: s_endpgm +define void @mac_v2f16_neg_c_unsafe_fp_math( + <2 x half> addrspace(1)* %r, + <2 x half> addrspace(1)* %a, + <2 x half> addrspace(1)* %b, + <2 x half> addrspace(1)* %c) #1 { +entry: + %a.val = load <2 x half>, <2 x half> addrspace(1)* %a + %b.val = load <2 x half>, <2 x half> addrspace(1)* %b + %c.val = load <2 x half>, <2 x half> addrspace(1)* %c + + %c.neg = fsub <2 x half> <half 0.0, half 0.0>, %c.val + %t.val = fmul <2 x half> %a.val, %b.val + %r.val = fadd <2 x half> %t.val, %c.neg + + store <2 x half> %r.val, <2 x half> addrspace(1)* %r + ret void +} + +attributes #0 = {"unsafe-fp-math"="false"} +attributes #1 = {"unsafe-fp-math"="true"} diff --git a/test/CodeGen/AMDGPU/v_madak_f16.ll b/test/CodeGen/AMDGPU/v_madak_f16.ll new file mode 100644 index 000000000000..fd5ad3e3d605 --- /dev/null +++ b/test/CodeGen/AMDGPU/v_madak_f16.ll @@ -0,0 +1,50 @@ +; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s +; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s + +; GCN-LABEL: {{^}}madak_f16 +; GCN: buffer_load_ushort v[[A_F16:[0-9]+]] +; GCN: buffer_load_ushort v[[B_F16:[0-9]+]] +; VI: v_madak_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]], 0x4900{{$}} +; VI: buffer_store_short v[[R_F16]] +; GCN: s_endpgm +define void @madak_f16( + half addrspace(1)* %r, + half addrspace(1)* %a, + half addrspace(1)* %b) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + + %t.val = fmul half %a.val, %b.val + %r.val = fadd half %t.val, 10.0 + + store half %r.val, half addrspace(1)* %r + ret void +} + +; GCN-LABEL: {{^}}madak_f16_use_2 +; SI: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; SI: v_mac_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mad_f16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; VI: v_mac_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} +; GCN: s_endpgm +define void @madak_f16_use_2( + half addrspace(1)* %r0, + half addrspace(1)* %r1, + half addrspace(1)* %a, + half addrspace(1)* %b, + half addrspace(1)* %c) { +entry: + %a.val = load half, half addrspace(1)* %a + %b.val = load half, half addrspace(1)* %b + %c.val = load half, half addrspace(1)* %c + + %t0.val = fmul half %a.val, %b.val + %t1.val = fmul half %a.val, %c.val + %r0.val = fadd half %t0.val, 10.0 + %r1.val = fadd half %t1.val, 10.0 + + store half %r0.val, half addrspace(1)* %r0 + store half %r1.val, half addrspace(1)* %r1 + ret void +} diff --git a/test/CodeGen/AMDGPU/valu-i1.ll b/test/CodeGen/AMDGPU/valu-i1.ll index 02a1278f76cb..e64f8467240a 100644 --- a/test/CodeGen/AMDGPU/valu-i1.ll +++ b/test/CodeGen/AMDGPU/valu-i1.ll @@ -2,11 +2,33 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone -; SI-LABEL: @test_if +; SI-LABEL: {{^}}test_if: ; Make sure the i1 values created by the cfg structurizer pass are ; moved using VALU instructions + + +; waitcnt should be inserted after exec modification +; SI: v_cmp_lt_i32_e32 vcc, 0, +; SI-NEXT: s_and_saveexec_b64 [[SAVE1:s\[[0-9]+:[0-9]+\]]], vcc +; SI-NEXT: s_xor_b64 [[SAVE2:s\[[0-9]+:[0-9]+\]]], exec, [[SAVE1]] +; SI-NEXT: s_waitcnt lgkmcnt(0) +; SI-NEXT: ; mask branch [[FLOW_BB:BB[0-9]+_[0-9]+]] +; SI-NEXT: s_cbranch_execz [[FLOW_BB]] + +; SI-NEXT: BB{{[0-9]+}}_1: ; %LeafBlock3 ; SI-NOT: s_mov_b64 s[{{[0-9]:[0-9]}}], -1 ; SI: v_mov_b32_e32 v{{[0-9]}}, -1 +; SI: s_and_saveexec_b64 +; SI-NEXT: s_xor_b64 +; SI-NEXT: ; mask branch + +; v_mov should be after exec modification +; SI: [[FLOW_BB]]: +; SI-NEXT: s_or_saveexec_b64 [[SAVE3:s\[[0-9]+:[0-9]+\]]], [[SAVE2]] +; SI-NEXT: v_mov_b32_e32 v{{[0-9]+}} +; SI-NEXT: s_xor_b64 exec, exec, [[SAVE3]] +; SI-NEXT: ; mask branch +; define void @test_if(i32 %b, i32 addrspace(1)* %src, i32 addrspace(1)* %dst) #1 { entry: %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone @@ -17,12 +39,12 @@ entry: case0: %arrayidx1 = getelementptr i32, i32 addrspace(1)* %dst, i32 %b - store i32 0, i32 addrspace(1)* %arrayidx1, align 4 + store i32 13, i32 addrspace(1)* %arrayidx1, align 4 br label %end case1: %arrayidx5 = getelementptr i32, i32 addrspace(1)* %dst, i32 %b - store i32 1, i32 addrspace(1)* %arrayidx5, align 4 + store i32 17, i32 addrspace(1)* %arrayidx5, align 4 br label %end default: @@ -31,11 +53,11 @@ default: br i1 %cmp8, label %if, label %else if: - store i32 2, i32 addrspace(1)* %arrayidx10, align 4 + store i32 19, i32 addrspace(1)* %arrayidx10, align 4 br label %end else: - store i32 3, i32 addrspace(1)* %arrayidx10, align 4 + store i32 21, i32 addrspace(1)* %arrayidx10, align 4 br label %end end: @@ -43,11 +65,11 @@ end: } ; SI-LABEL: @simple_test_v_if -; SI: v_cmp_ne_i32_e32 vcc, 0, v{{[0-9]+}} +; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}} ; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc ; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]] -; SI: ; BB#1 +; SI: BB{{[0-9]+_[0-9]+}}: ; SI: buffer_store_dword ; SI: s_endpgm @@ -68,8 +90,8 @@ exit: ret void } -; SI-LABEL: @simple_test_v_loop -; SI: v_cmp_ne_i32_e32 vcc, 0, v{{[0-9]+}} +; SI-LABEL: {{^}}simple_test_v_loop: +; SI: v_cmp_ne_u32_e32 vcc, 0, v{{[0-9]+}} ; SI: s_and_saveexec_b64 [[BR_SREG:s\[[0-9]+:[0-9]+\]]], vcc ; SI: s_xor_b64 [[BR_SREG]], exec, [[BR_SREG]] ; SI: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]] @@ -79,9 +101,8 @@ exit: ; SI: [[LABEL_LOOP:BB[0-9]+_[0-9]+]]: ; SI: buffer_load_dword ; SI-DAG: buffer_store_dword -; SI-DAG: v_cmp_eq_i32_e32 vcc, -; SI-DAG: s_and_b64 vcc, exec, vcc -; SI: s_cbranch_vccz [[LABEL_LOOP]] +; SI-DAG: s_cmpk_eq_i32 s{{[0-9]+}}, 0x100 +; SI: s_cbranch_scc0 [[LABEL_LOOP]] ; SI: [[LABEL_EXIT]]: ; SI: s_endpgm @@ -106,7 +127,7 @@ exit: ret void } -; SI-LABEL: @multi_vcond_loop +; SI-LABEL: {{^}}multi_vcond_loop: ; Load loop limit from buffer ; Branch to exit if uniformly not taken @@ -118,7 +139,7 @@ exit: ; SI: s_cbranch_execz [[LABEL_EXIT:BB[0-9]+_[0-9]+]] ; Initialize inner condition to false -; SI: ; BB#1: +; SI: BB{{[0-9]+_[0-9]+}}: ; %bb10.preheader ; SI: s_mov_b64 [[ZERO:s\[[0-9]+:[0-9]+\]]], 0{{$}} ; SI: s_mov_b64 [[COND_STATE:s\[[0-9]+:[0-9]+\]]], [[ZERO]] @@ -126,23 +147,24 @@ exit: ; SI: [[LABEL_LOOP:BB[0-9]+_[0-9]+]]: ; SI: buffer_load_dword [[B:v[0-9]+]] ; SI: buffer_load_dword [[A:v[0-9]+]] -; SI-DAG: v_cmp_ne_i32_e64 [[NEG1_CHECK_0:s\[[0-9]+:[0-9]+\]]], -1, [[A]] -; SI-DAG: v_cmp_ne_i32_e32 [[NEG1_CHECK_1:vcc]], -1, [[B]] +; SI-DAG: v_cmp_ne_u32_e64 [[NEG1_CHECK_0:s\[[0-9]+:[0-9]+\]]], -1, [[A]] +; SI-DAG: v_cmp_ne_u32_e32 [[NEG1_CHECK_1:vcc]], -1, [[B]] ; SI: s_and_b64 [[ORNEG1:s\[[0-9]+:[0-9]+\]]], [[NEG1_CHECK_1]], [[NEG1_CHECK_0]] ; SI: s_and_saveexec_b64 [[ORNEG2:s\[[0-9]+:[0-9]+\]]], [[ORNEG1]] -; SI: s_xor_b64 [[ORNEG2]], exec, [[ORNEG2]] +; SI: s_xor_b64 [[ORNEG3:s\[[0-9]+:[0-9]+\]]], exec, [[ORNEG2]] ; SI: s_cbranch_execz [[LABEL_FLOW:BB[0-9]+_[0-9]+]] -; SI: BB#3: +; SI: BB{{[0-9]+_[0-9]+}}: ; %bb20 ; SI: buffer_store_dword ; SI: v_cmp_ge_i64_e32 [[CMP:s\[[0-9]+:[0-9]+\]|vcc]] ; SI: s_or_b64 [[TMP:s\[[0-9]+:[0-9]+\]]], [[CMP]], [[COND_STATE]] ; SI: [[LABEL_FLOW]]: -; SI: s_or_b64 exec, exec, [[ORNEG2]] -; SI: s_or_b64 [[COND_STATE]], [[ORNEG2]], [[TMP]] -; SI: s_andn2_b64 exec, exec, [[COND_STATE]] -; SI: s_cbranch_execnz [[LABEL_LOOP]] +; SI-NEXT: ; in Loop: Header=[[LABEL_LOOP]] +; SI-NEXT: s_or_b64 exec, exec, [[ORNEG3]] +; SI-NEXT: s_or_b64 [[COND_STATE]], [[ORNEG3]], [[TMP]] +; SI-NEXT: s_andn2_b64 exec, exec, [[COND_STATE]] +; SI-NEXT: s_cbranch_execnz [[LABEL_LOOP]] ; SI: BB#5 ; SI: s_or_b64 exec, exec, [[COND_STATE]] diff --git a/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir b/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir new file mode 100644 index 000000000000..03e473e3a0c0 --- /dev/null +++ b/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir @@ -0,0 +1,177 @@ +# RUN: llc -run-pass si-insert-waits -march=amdgcn -mcpu=tahiti -o - %s | FileCheck %s +--- | + + define void @vccz_corrupt_workaround(float %cond, i32 addrspace(1)* %out) #0 { + entry: + %cmp0 = fcmp oeq float %cond, 0.000000e+00 + br i1 %cmp0, label %if, label %else, !structurizecfg.uniform !0, !amdgpu.uniform !0 + + else: ; preds = %entry + store volatile i32 100, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + if: ; preds = %entry + store volatile i32 9, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + done: ; preds = %if, %else + %value = phi i32 [ 0, %if ], [ 1, %else ] + store i32 %value, i32 addrspace(1)* %out + ret void + } + + define void @vccz_corrupt_undef_vcc(float %cond, i32 addrspace(1)* %out) #0 { + entry: + br i1 undef, label %if, label %else, !structurizecfg.uniform !0, !amdgpu.uniform !0 + + else: ; preds = %entry + store volatile i32 100, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + if: ; preds = %entry + store volatile i32 9, i32 addrspace(1)* undef + br label %done, !structurizecfg.uniform !0 + + done: ; preds = %if, %else + %value = phi i32 [ 0, %if ], [ 1, %else ] + store i32 %value, i32 addrspace(1)* %out + ret void + } + + attributes #0 = { nounwind } + attributes #1 = { readnone } + + !0 = !{} + +... +--- +# CHECK-LABEL: name: vccz_corrupt_workaround +# CHECK: %vcc = V_CMP_EQ_F32 +# CHECK-NEXT: %vcc = S_MOV_B64 %vcc +# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2.else, implicit killed %vcc + +name: vccz_corrupt_workaround +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%sgpr0_sgpr1' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.entry: + successors: %bb.2.if, %bb.1.else + liveins: %sgpr0_sgpr1 + + %sgpr2 = S_LOAD_DWORD_IMM %sgpr0_sgpr1, 9, 0 :: (non-temporal dereferenceable invariant load 4 from `float addrspace(2)* undef`) + %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`) + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + %vcc = V_CMP_EQ_F32_e64 0, 0, 0, %sgpr2, 0, 0, implicit %exec + S_CBRANCH_VCCZ %bb.1.else, implicit killed %vcc + + bb.2.if: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 9, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 0, implicit %exec + S_BRANCH %bb.3.done + + bb.1.else: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 100, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 1, implicit %exec + + bb.3.done: + liveins: %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into %ir.out) + S_ENDPGM + +... +--- +# CHECK-LABEL: name: vccz_corrupt_undef_vcc +# CHECK: S_WAITCNT +# CHECK-NEXT: S_CBRANCH_VCCZ %bb.2.else, implicit undef %vcc + +name: vccz_corrupt_undef_vcc +alignment: 0 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: + - { reg: '%sgpr0_sgpr1' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 0 + offsetAdjustment: 0 + maxAlignment: 0 + adjustsStack: false + hasCalls: false + maxCallFrameSize: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false +body: | + bb.0.entry: + successors: %bb.2.if, %bb.1.else + liveins: %sgpr0_sgpr1 + + %sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed %sgpr0_sgpr1, 11, 0 :: (non-temporal dereferenceable invariant load 8 from `i64 addrspace(2)* undef`) + %sgpr7 = S_MOV_B32 61440 + %sgpr6 = S_MOV_B32 -1 + S_CBRANCH_VCCZ %bb.1.else, implicit undef %vcc + + bb.2.if: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 9, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 0, implicit %exec + S_BRANCH %bb.3.done + + bb.1.else: + successors: %bb.3.done + liveins: %sgpr6, %sgpr7, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %vgpr0 = V_MOV_B32_e32 100, implicit %exec + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit %exec :: (volatile store 4 into `i32 addrspace(1)* undef`) + %vgpr0 = V_MOV_B32_e32 1, implicit %exec + + bb.3.done: + liveins: %vgpr0, %sgpr0_sgpr1_sgpr2_sgpr3:0x00000003 + + %sgpr3 = S_MOV_B32 61440 + %sgpr2 = S_MOV_B32 -1 + BUFFER_STORE_DWORD_OFFSET killed %vgpr0, killed %sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, implicit %exec :: (store 4 into %ir.out) + S_ENDPGM + +... diff --git a/test/CodeGen/AMDGPU/vertex-fetch-encoding.ll b/test/CodeGen/AMDGPU/vertex-fetch-encoding.ll index fb6a17e67146..3d71062f1fba 100644 --- a/test/CodeGen/AMDGPU/vertex-fetch-encoding.ll +++ b/test/CodeGen/AMDGPU/vertex-fetch-encoding.ll @@ -1,25 +1,46 @@ -; RUN: llc < %s -march=r600 -show-mc-encoding -mcpu=barts | FileCheck --check-prefix=NI %s -; RUN: llc < %s -march=r600 -show-mc-encoding -mcpu=cayman | FileCheck --check-prefix=CM %s +; RUN: llc < %s -march=r600 -show-mc-encoding -mcpu=cypress | FileCheck --check-prefix=EG --check-prefix=FUNC %s +; RUN: llc < %s -march=r600 -show-mc-encoding -mcpu=barts | FileCheck --check-prefix=EG --check-prefix=FUNC %s +; RUN: llc < %s -march=r600 -show-mc-encoding -mcpu=cayman | FileCheck --check-prefix=CM --check-prefix=FUNC %s -; NI: {{^}}vtx_fetch32: -; NI: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0 ; encoding: [0x40,0x01,0x0[[GPR]],0x10,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x08,0x00 -; CM: {{^}}vtx_fetch32: -; CM: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0 ; encoding: [0x40,0x01,0x0[[GPR]],0x00,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x00,0x00 +; FUNC-LABEL: {{^}}vtx_fetch32: +; EG: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #1 ; encoding: [0x40,0x01,0x0[[GPR]],0x10,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x08,0x00 +; CM: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #1 ; encoding: [0x40,0x01,0x0[[GPR]],0x00,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x00,0x00 define void @vtx_fetch32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) { -entry: - %0 = load i32, i32 addrspace(1)* %in - store i32 %0, i32 addrspace(1)* %out + %v = load i32, i32 addrspace(1)* %in + store i32 %v, i32 addrspace(1)* %out ret void } -; NI: {{^}}vtx_fetch128: -; NI: VTX_READ_128 T[[DST:[0-9]]].XYZW, T[[SRC:[0-9]]].X, 0 ; encoding: [0x40,0x01,0x0[[SRC]],0x40,0x0[[DST]],0x10,0x8d,0x18,0x00,0x00,0x08,0x00 -; XXX: Add a case for Cayman when v4i32 stores are supported. +; FUNC-LABEL: {{^}}vtx_fetch128: +; EG: VTX_READ_128 T[[DST:[0-9]]].XYZW, T[[SRC:[0-9]]].X, 0, #1 ; encoding: [0x40,0x01,0x0[[SRC]],0x40,0x0[[DST]],0x10,0x8d,0x18,0x00,0x00,0x08,0x00 +; CM: VTX_READ_128 T[[DST:[0-9]]].XYZW, T[[SRC:[0-9]]].X, 0, #1 ; encoding: [0x40,0x01,0x0[[SRC]],0x00,0x0[[DST]],0x10,0x8d,0x18,0x00,0x00,0x00,0x00 define void @vtx_fetch128(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %in) { -entry: - %0 = load <4 x i32>, <4 x i32> addrspace(1)* %in - store <4 x i32> %0, <4 x i32> addrspace(1)* %out + %v = load <4 x i32>, <4 x i32> addrspace(1)* %in + store <4 x i32> %v, <4 x i32> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}vtx_fetch32_id3: +; EG: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #3 ; encoding: [0x40,0x03,0x0[[GPR]],0x10,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x08,0x00 +; CM: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #3 ; encoding: [0x40,0x03,0x0[[GPR]],0x00,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x00,0x00 + +define void @vtx_fetch32_id3(i32 addrspace(1)* %out, i32 addrspace(7)* %in) { + %v = load i32, i32 addrspace(7)* %in + store i32 %v, i32 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}vtx_fetch32_id2: +; EG: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #2 ; encoding: [0x40,0x02,0x0[[GPR]],0x10,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x08,0x00 +; CM: VTX_READ_32 T[[GPR:[0-9]]].X, T[[GPR]].X, 0, #2 ; encoding: [0x40,0x02,0x0[[GPR]],0x00,0x0[[GPR]],0xf0,0x5f,0x13,0x00,0x00,0x00,0x00 + +@t = internal addrspace(2) constant [4 x i32] [i32 0, i32 1, i32 2, i32 3] + +define void @vtx_fetch32_id2(i32 addrspace(1)* %out, i32 %in) { + %a = getelementptr inbounds [4 x i32], [4 x i32] addrspace(2)* @t, i32 0, i32 %in + %v = load i32, i32 addrspace(2)* %a + store i32 %v, i32 addrspace(1)* %out ret void } diff --git a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll index 6c33bc98c605..df7cf09e9933 100644 --- a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll +++ b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot-compute.ll @@ -29,15 +29,15 @@ ; GCN: buffer_store_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} ; 4-byte Folded Spill -; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_store_dword {{v[0-9]}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} +; GCN: buffer_store_dword {{v[0-9]}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_store_dword {{v[0-9]}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_store_dword {{v[0-9]}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_store_dword {{v[0-9]}}, off, s[12:15], s16 offset:{{[0-9]+}} -; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} -; GCN: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, s[12:15], s16 offen offset:{{[0-9]+}} +; GCN: buffer_load_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_load_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_load_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} +; GCN: buffer_load_dword {{v[0-9]+}}, off, s[12:15], s16 offset:{{[0-9]+}} ; GCN: NumVgprs: 256 ; GCN: ScratchSize: 1024 diff --git a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll index 7d97777a78bd..4de35b97aeab 100644 --- a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll +++ b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll @@ -12,16 +12,16 @@ ; GCN-LABEL: {{^}}main: -; GCN-DAG: s_mov_b32 s13, s12 -; GCN-DAG: s_mov_b32 s16, SCRATCH_RSRC_DWORD0 -; GCN-DAG: s_mov_b32 s17, SCRATCH_RSRC_DWORD1 -; GCN-DAG: s_mov_b32 s18, -1 -; SI-DAG: s_mov_b32 s19, 0xe8f000 -; VI-DAG: s_mov_b32 s19, 0xe80000 +; GCN-DAG: s_mov_b32 s11, s12 +; GCN-DAG: s_mov_b32 s12, SCRATCH_RSRC_DWORD0 +; GCN-DAG: s_mov_b32 s13, SCRATCH_RSRC_DWORD1 +; GCN-DAG: s_mov_b32 s14, -1 +; SI-DAG: s_mov_b32 s15, 0xe8f000 +; VI-DAG: s_mov_b32 s15, 0xe80000 -; s13 is offset system SGPR -; GCN: buffer_store_dword {{v[0-9]+}}, off, s[16:19], s13 offset:{{[0-9]+}} ; 16-byte Folded Spill -; GCN: buffer_load_dword v{{[0-9]+}}, off, s[16:19], s13 offset:{{[0-9]+}} ; 16-byte Folded Reload +; s11 is offset system SGPR +; GCN: buffer_store_dword {{v[0-9]+}}, off, s[12:15], s11 offset:{{[0-9]+}} ; 4-byte Folded Spill +; GCN: buffer_load_dword v{{[0-9]+}}, off, s[12:15], s11 offset:{{[0-9]+}} ; 4-byte Folded Reload ; GCN: NumVgprs: 256 ; GCN: ScratchSize: 1024 @@ -497,4 +497,5 @@ attributes #0 = { nounwind } attributes #1 = { nounwind readnone } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !2} +!2 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/wait.ll b/test/CodeGen/AMDGPU/wait.ll index 265774180a7f..152a474cde7e 100644 --- a/test/CodeGen/AMDGPU/wait.ll +++ b/test/CodeGen/AMDGPU/wait.ll @@ -82,4 +82,5 @@ attributes #1 = { convergent nounwind } attributes #2 = { nounwind readnone } !0 = !{!1, !1, i64 0, i32 1} -!1 = !{!"const", null} +!1 = !{!"const", !2} +!2 = !{!"tbaa root"} diff --git a/test/CodeGen/AMDGPU/waitcnt.mir b/test/CodeGen/AMDGPU/waitcnt.mir new file mode 100644 index 000000000000..cb5de6a2419d --- /dev/null +++ b/test/CodeGen/AMDGPU/waitcnt.mir @@ -0,0 +1,59 @@ +# RUN: llc -march=amdgcn -mcpu=fiji -run-pass si-insert-waits %s -o - | FileCheck %s + +--- | + define void @flat_zero_waitcnt(i32 addrspace(1)* %global4, + <4 x i32> addrspace(1)* %global16, + i32 addrspace(4)* %flat4, + <4 x i32> addrspace(4)* %flat16) { + ret void + } +... +--- + +# CHECK-LABEL: name: flat_zero_waitcnt + +# CHECK-LABEL: bb.0: +# CHECK: FLAT_LOAD_DWORD +# CHECK: FLAT_LOAD_DWORDX4 +# Global loads will return in order so we should: +# s_waitcnt vmcnt(1) lgkmcnt(0) +# CHECK-NEXT: S_WAITCNT 113 + +# CHECK-LABEL: bb.1: +# CHECK: FLAT_LOAD_DWORD +# CHECK: FLAT_LOAD_DWORDX4 +# The first load has no mem operand, so we should assume it accesses the flat +# address space. +# s_waitcnt vmcnt(0) lgkmcnt(0) +# CHECK-NEXT: S_WAITCNT 112 + +# CHECK-LABEL: bb.2: +# CHECK: FLAT_LOAD_DWORD +# CHECK: FLAT_LOAD_DWORDX4 +# One outstand loads access the flat address space. +# s_waitcnt vmcnt(0) lgkmcnt(0) +# CHECK-NEXT: S_WAITCNT 112 + +name: flat_zero_waitcnt + +body: | + bb.0: + successors: %bb.1 + %vgpr0 = FLAT_LOAD_DWORD %vgpr1_vgpr2, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4 from %ir.global4) + %vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 %vgpr7_vgpr8, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 16 from %ir.global16) + %vgpr0 = V_MOV_B32_e32 %vgpr1, implicit %exec + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2 + %vgpr0 = FLAT_LOAD_DWORD %vgpr1_vgpr2, 0, 0, 0, implicit %exec, implicit %flat_scr + %vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 %vgpr7_vgpr8, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 16 from %ir.global16) + %vgpr0 = V_MOV_B32_e32 %vgpr1, implicit %exec + S_BRANCH %bb.2 + + bb.2: + %vgpr0 = FLAT_LOAD_DWORD %vgpr1_vgpr2, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 4 from %ir.flat4) + %vgpr3_vgpr4_vgpr5_vgpr6 = FLAT_LOAD_DWORDX4 %vgpr7_vgpr8, 0, 0, 0, implicit %exec, implicit %flat_scr :: (load 16 from %ir.flat16) + %vgpr0 = V_MOV_B32_e32 %vgpr1, implicit %exec + S_ENDPGM +... diff --git a/test/CodeGen/AMDGPU/wqm.ll b/test/CodeGen/AMDGPU/wqm.ll index 41e426457889..3f7b2b284c53 100644 --- a/test/CodeGen/AMDGPU/wqm.ll +++ b/test/CodeGen/AMDGPU/wqm.ll @@ -7,8 +7,8 @@ ;CHECK-NOT: s_wqm define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, <4 x i32> %c) { main_body: - %tex = call <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) - call void @llvm.amdgcn.image.store.v4i32(<4 x float> %tex, <4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + %tex = call <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float> %tex, <4 x i32> %c, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0) ret <4 x float> %tex } @@ -37,8 +37,8 @@ main_body: ;CHECK-NEXT: ; %main_body ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec ;CHECK-NEXT: s_wqm_b64 exec, exec -;CHECK: image_sample ;CHECK: s_and_b64 exec, exec, [[ORIG]] +;CHECK: image_sample ;CHECK: store ;CHECK-NOT: exec ;CHECK: .size test3 @@ -63,7 +63,8 @@ main_body: ;CHECK: s_and_b64 exec, exec, [[ORIG]] ;CHECK: store ;CHECK: s_wqm_b64 exec, exec -;CHECK: image_sample v[0:3], [[MUL]], s[0:7], s[8:11] dmask:0xf +;CHECK: image_sample +;CHECK: image_sample define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) { main_body: %c.1 = mul i32 %c, %d @@ -71,7 +72,9 @@ main_body: call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i1 0, i1 0) %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - ret <4 x float> %tex + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + ret <4 x float> %dtex } ; Check a case of one branch of an if-else requiring WQM, the other requiring @@ -91,6 +94,7 @@ main_body: ;CHECK: s_mov_b64 exec, [[SAVED]] ;CHECK: %IF ;CHECK: image_sample +;CHECK: image_sample define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) { main_body: %cmp = icmp eq i32 %z, 0 @@ -98,7 +102,9 @@ main_body: IF: %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - %data.if = extractelement <4 x float> %tex, i32 0 + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %data.if = extractelement <4 x float> %dtex, i32 0 br label %END ELSE: @@ -118,13 +124,14 @@ END: ;CHECK-NEXT: s_wqm_b64 exec, exec ;CHECK: %IF ;CHECK: image_sample +;CHECK: image_sample ;CHECK: %Flow ;CHECK-NEXT: s_or_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], ;CHECK-NEXT: s_and_b64 exec, exec, [[ORIG]] ;CHECK-NEXT: s_and_b64 [[SAVED]], exec, [[SAVED]] ;CHECK-NEXT: s_xor_b64 exec, exec, [[SAVED]] ;CHECK-NEXT: mask branch [[END_BB:BB[0-9]+_[0-9]+]] -;CHECK-NEXT: ; BB#3: ; %ELSE +;CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %ELSE ;CHECK: store_dword ;CHECK: [[END_BB]]: ; %END ;CHECK: s_or_b64 exec, exec, @@ -137,7 +144,9 @@ main_body: IF: %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %c, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - %data.if = extractelement <4 x float> %tex, i32 0 + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %data.if = extractelement <4 x float> %dtex, i32 0 br label %END ELSE: @@ -203,35 +212,27 @@ END: ;CHECK-NEXT: s_wqm_b64 exec, exec ;CHECK: image_sample ;CHECK: s_and_b64 exec, exec, [[ORIG]] -;CHECK: store -;CHECK: load -;CHECK: store +;CHECK: image_sample ;CHECK: v_cmp -define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) { +;CHECK: store +define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, i32 %coord) { main_body: %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - %tex.1 = extractelement <4 x float> %tex, i32 0 - - %idx.1 = extractelement <3 x i32> %idx, i32 0 - %data.1 = extractelement <2 x float> %data, i32 0 - call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0) + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %dtex.1 = extractelement <4 x float> %dtex, i32 0 - %idx.2 = extractelement <3 x i32> %idx, i32 1 - %z = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i1 0, i1 0) + call void @llvm.amdgcn.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) - %idx.3 = extractelement <3 x i32> %idx, i32 2 - %data.3 = extractelement <2 x float> %data, i32 1 - call void @llvm.amdgcn.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i1 0, i1 0) - - %cc = fcmp ogt float %z, 0.0 + %cc = fcmp ogt float %dtex.1, 0.0 br i1 %cc, label %IF, label %ELSE IF: - %tex.IF = fmul float %tex.1, 3.0 + %tex.IF = fmul float %dtex.1, 3.0 br label %END ELSE: - %tex.ELSE = fmul float %tex.1, 4.0 + %tex.ELSE = fmul float %dtex.1, 4.0 br label %END END: @@ -246,12 +247,13 @@ END: ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec ;CHECK-NEXT: s_wqm_b64 exec, exec ;CHECK: %IF -;CHECK: load ;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]] +;CHECK: load ;CHECK: store ;CHECK: s_mov_b64 exec, [[SAVE]] ;CHECK: %END ;CHECK: image_sample +;CHECK: image_sample define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %coord, i32 %y, float %z) { main_body: %cond = icmp eq i32 %y, 0 @@ -264,7 +266,9 @@ IF: END: %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - ret <4 x float> %tex + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + ret <4 x float> %dtex } ; Kill is performed in WQM mode so that uniform kill behaves correctly ... @@ -273,8 +277,8 @@ END: ;CHECK-NEXT: ; %main_body ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec ;CHECK-NEXT: s_wqm_b64 exec, exec -;CHECK: image_sample ;CHECK: s_and_b64 exec, exec, [[ORIG]] +;CHECK: image_sample ;CHECK: buffer_store_dword ;CHECK: s_wqm_b64 exec, exec ;CHECK: v_cmpx_ @@ -297,7 +301,9 @@ main_body: call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0) %tex2 = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - %out = fadd <4 x float> %tex, %tex2 + %tex2.1 = bitcast <4 x float> %tex2 to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex2.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %out = fadd <4 x float> %tex, %dtex ret <4 x float> %out } @@ -310,18 +316,21 @@ main_body: ; CHECK: s_wqm_b64 exec, exec ; CHECK: image_sample ; CHECK: s_and_b64 exec, exec, [[ORIG]] +; CHECK: image_sample ; CHECK: buffer_store_dword ; CHECK-NOT: wqm ; CHECK: v_cmpx_ define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, i32 %coord, i32 %coord2, float %z) { main_body: %tex = call <4 x float> @llvm.SI.image.sample.i32(i32 %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + %tex.1 = bitcast <4 x float> %tex to <4 x i32> + %dtex = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %tex.1, <8 x i32> %rsrc, <4 x i32> %sampler, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0) call void @llvm.AMDGPU.kill(float %z) - ret <4 x float> %tex + ret <4 x float> %dtex } ; Check prolog shaders. @@ -344,31 +353,31 @@ main_body: ; CHECK: s_and_b64 exec, exec, [[LIVE]] ; CHECK: image_store ; CHECK: s_wqm_b64 exec, exec -; CHECK: v_mov_b32_e32 [[CTR:v[0-9]+]], -2 -; CHECK: s_branch [[LOOPHDR:BB[0-9]+_[0-9]+]] +; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0 +; CHECK-DAG: v_mov_b32_e32 [[SEVEN:v[0-9]+]], 0x40e00000 -; CHECK: [[LOOPHDR]]: ; %loop -; CHECK: v_add_i32_e32 [[CTR]], vcc, 2, [[CTR]] -; CHECK: v_cmp_lt_i32_e32 vcc, 7, [[CTR]] -; CHECK: s_cbranch_vccz +; CHECK: [[LOOPHDR:BB[0-9]+_[0-9]+]]: ; %body +; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]] +; CHECK: v_cmp_lt_f32_e32 vcc, [[SEVEN]], [[CTR]] +; CHECK: s_cbranch_vccz [[LOOPHDR]] ; CHECK: ; %break ; CHECK: ; return define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind { entry: - call void @llvm.amdgcn.image.store.v4i32(<4 x float> %in, <4 x i32> undef, <8 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0) + call void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float> %in, <4 x i32> undef, <8 x i32> undef, i32 15, i1 0, i1 0, i1 0, i1 0) br label %loop loop: - %ctr.iv = phi i32 [ 0, %entry ], [ %ctr.next, %body ] + %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ] %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ] - %cc = icmp sgt i32 %ctr.iv, 7 + %cc = fcmp ogt float %ctr.iv, 7.0 br i1 %cc, label %break, label %body body: %c.i = bitcast <4 x float> %c.iv to <4 x i32> %c.next = call <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32> %c.i, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) - %ctr.next = add i32 %ctr.iv, 2 + %ctr.next = fadd float %ctr.iv, 2.0 br label %loop break: @@ -385,14 +394,14 @@ break: ; CHECK: s_and_b64 exec, exec, [[LIVE]] ; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0 ; CHECK: s_wqm_b64 exec, exec -; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen +; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+$}} ; CHECK: s_and_b64 exec, exec, [[LIVE]] ; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen ; CHECK: s_wqm_b64 exec, exec ; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen -; CHECK: image_sample ; CHECK: s_and_b64 exec, exec, [[LIVE]] +; CHECK: image_sample ; CHECK: buffer_store_dwordx4 define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind { entry: @@ -449,21 +458,58 @@ entry: br i1 %cc, label %if, label %else if: - store volatile <4 x float> %dtex, <4 x float>* undef + store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef unreachable else: ret <4 x float> %dtex } -declare void @llvm.amdgcn.image.store.v4i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 +; Test awareness that s_wqm_b64 clobbers SCC. +; +; CHECK-LABEL: {{^}}test_scc: +; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec +; CHECK: s_wqm_b64 exec, exec +; CHECK: s_cmp_ +; CHECK-NEXT: s_cbranch_scc +; CHECK: ; %if +; CHECK: s_and_b64 exec, exec, [[ORIG]] +; CHECK: image_sample +; CHECK: ; %else +; CHECK: s_and_b64 exec, exec, [[ORIG]] +; CHECK: image_sample +; CHECK: ; %end +define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 { +main_body: + %cc = icmp sgt i32 %sel, 0 + br i1 %cc, label %if, label %else + +if: + %r.if = call <4 x float> @llvm.SI.image.sample.i32(i32 0, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + br label %end + +else: + %r.else = call <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32> <i32 0, i32 1>, <8 x i32> undef, <4 x i32> undef, i32 15, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0) + br label %end + +end: + %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ] + + call void @llvm.amdgcn.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0) + + ret <4 x float> %r +} + + +declare void @llvm.amdgcn.image.store.v4f32.v4i32.v8i32(<4 x float>, <4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #1 declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #1 declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #1 -declare <4 x float> @llvm.amdgcn.image.load.v4i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #2 +declare <4 x float> @llvm.amdgcn.image.load.v4f32.v4i32.v8i32(<4 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #2 declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #2 declare <4 x float> @llvm.SI.image.sample.i32(i32, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #3 +declare <4 x float> @llvm.SI.image.sample.v2i32(<2 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #3 declare <4 x float> @llvm.SI.image.sample.v4i32(<4 x i32>, <8 x i32>, <4 x i32>, i32, i32, i32, i32, i32, i32, i32, i32) #3 declare void @llvm.AMDGPU.kill(float) diff --git a/test/CodeGen/AMDGPU/xfail.r600.bitcast.ll b/test/CodeGen/AMDGPU/xfail.r600.bitcast.ll new file mode 100644 index 000000000000..babae9ead27c --- /dev/null +++ b/test/CodeGen/AMDGPU/xfail.r600.bitcast.ll @@ -0,0 +1,46 @@ +; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s +; XFAIL: * + +; This is the failing part of the r600 bitacts tests + +; TODO: enable doubles +; FUNC-LABEL: {{^}}bitcast_f64_to_v2i32: +define void @bitcast_f64_to_v2i32(<2 x i32> addrspace(1)* %out, double addrspace(1)* %in) { + %val = load double, double addrspace(1)* %in, align 8 + %add = fadd double %val, 4.0 + %bc = bitcast double %add to <2 x i32> + store <2 x i32> %bc, <2 x i32> addrspace(1)* %out, align 8 + ret void +} + +; FUNC-LABEL: {{^}}bitcast_v2i64_to_v2f64: +define void @bitcast_v2i64_to_v2f64(i32 %cond, <2 x double> addrspace(1)* %out, <2 x i64> %value) { +entry: + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %if, label %end + +if: + %cast = bitcast <2 x i64> %value to <2 x double> + br label %end + +end: + %phi = phi <2 x double> [zeroinitializer, %entry], [%cast, %if] + store <2 x double> %phi, <2 x double> addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}bitcast_v2f64_to_v2i64: +define void @bitcast_v2f64_to_v2i64(i32 %cond, <2 x i64> addrspace(1)* %out, <2 x double> %value) { +entry: + %cmp0 = icmp eq i32 %cond, 0 + br i1 %cmp0, label %if, label %end + +if: + %cast = bitcast <2 x double> %value to <2 x i64> + br label %end + +end: + %phi = phi <2 x i64> [zeroinitializer, %entry], [%cast, %if] + store <2 x i64> %phi, <2 x i64> addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/xor.ll b/test/CodeGen/AMDGPU/xor.ll index 202170d6e229..53f4c0a91742 100644 --- a/test/CodeGen/AMDGPU/xor.ll +++ b/test/CodeGen/AMDGPU/xor.ll @@ -171,3 +171,81 @@ endif: store i64 %3, i64 addrspace(1)* %out ret void } + +; FUNC-LABEL: {{^}}scalar_xor_literal_i64: +; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-DAG: s_xor_b32 s[[RES_HI:[0-9]+]], s[[HI]], 0xf237b +; SI-DAG: s_xor_b32 s[[RES_LO:[0-9]+]], s[[LO]], 0x3039 +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_LO]] +; SI-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[RES_HI]] +define void @scalar_xor_literal_i64(i64 addrspace(1)* %out, i64 %a) { + %or = xor i64 %a, 4261135838621753 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}scalar_xor_literal_multi_use_i64: +; SI: s_load_dwordx2 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-DAG: s_mov_b32 s[[K_HI:[0-9]+]], 0xf237b +; SI-DAG: s_movk_i32 s[[K_LO:[0-9]+]], 0x3039 +; SI: s_xor_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[}}[[K_LO]]:[[K_HI]]{{\]}} + +; SI: s_add_u32 s{{[0-9]+}}, s{{[0-9]+}}, s[[K_LO]] +; SI: s_addc_u32 s{{[0-9]+}}, s{{[0-9]+}}, s[[K_HI]] +define void @scalar_xor_literal_multi_use_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) { + %or = xor i64 %a, 4261135838621753 + store i64 %or, i64 addrspace(1)* %out + + %foo = add i64 %b, 4261135838621753 + store volatile i64 %foo, i64 addrspace(1)* undef + ret void +} + +; FUNC-LABEL: {{^}}scalar_xor_inline_imm_i64: +; SI: s_load_dwordx2 s{{\[}}[[VAL_LO:[0-9]+]]:[[VAL_HI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI-NOT: xor_b32 +; SI: s_xor_b32 s[[VAL_LO]], s[[VAL_LO]], 63 +; SI-NOT: xor_b32 +; SI: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[VAL_LO]] +; SI-NOT: xor_b32 +; SI: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[VAL_HI]] +; SI-NOT: xor_b32 +; SI: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}} +define void @scalar_xor_inline_imm_i64(i64 addrspace(1)* %out, i64 %a) { + %or = xor i64 %a, 63 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}scalar_xor_neg_inline_imm_i64: +; SI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} +; SI: s_xor_b64 [[VAL]], [[VAL]], -8 +define void @scalar_xor_neg_inline_imm_i64(i64 addrspace(1)* %out, i64 %a) { + %or = xor i64 %a, -8 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}vector_xor_i64_neg_inline_imm: +; SI: buffer_load_dwordx2 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, +; SI: v_xor_b32_e32 {{v[0-9]+}}, -8, v[[LO_VREG]] +; SI: v_xor_b32_e32 {{v[0-9]+}}, -1, {{.*}} +; SI: s_endpgm +define void @vector_xor_i64_neg_inline_imm(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { + %loada = load i64, i64 addrspace(1)* %a, align 8 + %or = xor i64 %loada, -8 + store i64 %or, i64 addrspace(1)* %out + ret void +} + +; FUNC-LABEL: {{^}}vector_xor_literal_i64: +; SI-DAG: buffer_load_dwordx2 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, +; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0xdf77987f, v[[LO_VREG]] +; SI-DAG: v_xor_b32_e32 {{v[0-9]+}}, 0x146f, v[[HI_VREG]] +; SI: s_endpgm +define void @vector_xor_literal_i64(i64 addrspace(1)* %out, i64 addrspace(1)* %a, i64 addrspace(1)* %b) { + %loada = load i64, i64 addrspace(1)* %a, align 8 + %or = xor i64 %loada, 22470723082367 + store i64 %or, i64 addrspace(1)* %out + ret void +} diff --git a/test/CodeGen/AMDGPU/zero_extend.ll b/test/CodeGen/AMDGPU/zero_extend.ll index c3b76da5f778..b30cb73f6dac 100644 --- a/test/CodeGen/AMDGPU/zero_extend.ll +++ b/test/CodeGen/AMDGPU/zero_extend.ll @@ -2,39 +2,58 @@ ; RUN: llc < %s -march=amdgcn -mcpu=tonga -verify-machineinstrs | FileCheck %s --check-prefix=SI ; RUN: llc < %s -march=r600 -mcpu=redwood | FileCheck %s --check-prefix=R600 -; R600: {{^}}test: +; R600: {{^}}s_mad_zext_i32_to_i64: ; R600: MEM_RAT_CACHELESS STORE_RAW ; R600: MEM_RAT_CACHELESS STORE_RAW -; SI: {{^}}test: +; SI: {{^}}s_mad_zext_i32_to_i64: ; SI: v_mov_b32_e32 v[[V_ZERO:[0-9]]], 0{{$}} ; SI: buffer_store_dwordx2 v[0:[[V_ZERO]]{{\]}} -define void @test(i64 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) { +define void @s_mad_zext_i32_to_i64(i64 addrspace(1)* %out, i32 %a, i32 %b, i32 %c) #0 { entry: - %0 = mul i32 %a, %b - %1 = add i32 %0, %c - %2 = zext i32 %1 to i64 - store i64 %2, i64 addrspace(1)* %out + %tmp0 = mul i32 %a, %b + %tmp1 = add i32 %tmp0, %c + %tmp2 = zext i32 %tmp1 to i64 + store i64 %tmp2, i64 addrspace(1)* %out ret void } -; SI-LABEL: {{^}}testi1toi32: +; SI-LABEL: {{^}}s_cmp_zext_i1_to_i32 ; SI: v_cndmask_b32 -define void @testi1toi32(i32 addrspace(1)* %out, i32 %a, i32 %b) { +define void @s_cmp_zext_i1_to_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) #0 { entry: - %0 = icmp eq i32 %a, %b - %1 = zext i1 %0 to i32 - store i32 %1, i32 addrspace(1)* %out + %tmp0 = icmp eq i32 %a, %b + %tmp1 = zext i1 %tmp0 to i32 + store i32 %tmp1, i32 addrspace(1)* %out ret void } -; SI-LABEL: {{^}}zext_i1_to_i64: +; SI-LABEL: {{^}}s_arg_zext_i1_to_i64: +define void @s_arg_zext_i1_to_i64(i64 addrspace(1)* %out, i1 zeroext %arg) #0 { + %ext = zext i1 %arg to i64 + store i64 %ext, i64 addrspace(1)* %out, align 8 + ret void +} + +; SI-LABEL: {{^}}s_cmp_zext_i1_to_i64: ; SI: s_mov_b32 s{{[0-9]+}}, 0 -; SI: v_cmp_eq_i32 +; SI: v_cmp_eq_u32 ; SI: v_cndmask_b32 -define void @zext_i1_to_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) nounwind { +define void @s_cmp_zext_i1_to_i64(i64 addrspace(1)* %out, i32 %a, i32 %b) #0 { %cmp = icmp eq i32 %a, %b %ext = zext i1 %cmp to i64 store i64 %ext, i64 addrspace(1)* %out, align 8 ret void } + +; SI-LABEL: {{^}}s_cmp_zext_i1_to_i16 +; SI: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc +; SI: buffer_store_short [[RESULT]] +define void @s_cmp_zext_i1_to_i16(i16 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) #0 { + %tmp0 = icmp eq i16 %a, %b + %tmp1 = zext i1 %tmp0 to i16 + store i16 %tmp1, i16 addrspace(1)* %out + ret void +} + +attributes #0 = { nounwind } |