diff options
Diffstat (limited to 'test/Transforms/LoopVectorize')
85 files changed, 3055 insertions, 254 deletions
diff --git a/test/Transforms/LoopVectorize/12-12-11-if-conv.ll b/test/Transforms/LoopVectorize/12-12-11-if-conv.ll index bab6300f2e7f9..1e1396f800851 100644 --- a/test/Transforms/LoopVectorize/12-12-11-if-conv.ll +++ b/test/Transforms/LoopVectorize/12-12-11-if-conv.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK: icmp eq <4 x i32> ;CHECK: select <4 x i1> ;CHECK: ret i32 @@ -30,7 +30,7 @@ if.then: ; preds = %for.body if.end: ; preds = %for.body, %if.then %z.0 = phi i32 [ %add1, %if.then ], [ 9, %for.body ] store i32 %z.0, i32* %arrayidx, align 4 - %indvars.iv.next = add i64 %indvars.iv, 1 + %indvars.iv.next = add nsw i64 %indvars.iv, 1 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 %exitcond = icmp eq i32 %lftr.wideiv, %x br i1 %exitcond, label %for.end, label %for.body diff --git a/test/Transforms/LoopVectorize/ARM/arm-unroll.ll b/test/Transforms/LoopVectorize/ARM/arm-unroll.ll index c8d307f5d4438..39363ab2d802b 100644 --- a/test/Transforms/LoopVectorize/ARM/arm-unroll.ll +++ b/test/Transforms/LoopVectorize/ARM/arm-unroll.ll @@ -4,11 +4,11 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32" target triple = "thumbv7-apple-ios3.0.0" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK: load <4 x i32> ;CHECK-NOT: load <4 x i32> ;CHECK: ret -;SWIFT: @foo +;SWIFT-LABEL: @foo( ;SWIFT: load <4 x i32> ;SWIFT: load <4 x i32> ;SWIFT: ret diff --git a/test/Transforms/LoopVectorize/ARM/gather-cost.ll b/test/Transforms/LoopVectorize/ARM/gather-cost.ll new file mode 100644 index 0000000000000..239a28fa6c7c0 --- /dev/null +++ b/test/Transforms/LoopVectorize/ARM/gather-cost.ll @@ -0,0 +1,88 @@ +; RUN: opt -loop-vectorize -mtriple=thumbv7s-apple-ios6.0.0 -S < %s | FileCheck %s + +target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32" + +@kernel = global [512 x float] zeroinitializer, align 4 +@kernel2 = global [512 x float] zeroinitializer, align 4 +@kernel3 = global [512 x float] zeroinitializer, align 4 +@kernel4 = global [512 x float] zeroinitializer, align 4 +@src_data = global [1536 x float] zeroinitializer, align 4 +@r_ = global i8 0, align 4 +@g_ = global i8 0, align 4 +@b_ = global i8 0, align 4 + +; We don't want to vectorize most loops containing gathers because they are +; expensive. This function represents a point where vectorization starts to +; become beneficial. +; Make sure we are conservative and don't vectorize it. +; CHECK-NOT: <2 x float> +; CHECK-NOT: <4 x float> + +define void @_Z4testmm(i32 %size, i32 %offset) { +entry: + %cmp53 = icmp eq i32 %size, 0 + br i1 %cmp53, label %for.end, label %for.body.lr.ph + +for.body.lr.ph: + br label %for.body + +for.body: + %r.057 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add10, %for.body ] + %g.056 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add20, %for.body ] + %v.055 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ] + %b.054 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add30, %for.body ] + %add = add i32 %v.055, %offset + %mul = mul i32 %add, 3 + %arrayidx = getelementptr inbounds [1536 x float]* @src_data, i32 0, i32 %mul + %0 = load float* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds [512 x float]* @kernel, i32 0, i32 %v.055 + %1 = load float* %arrayidx2, align 4 + %mul3 = fmul fast float %0, %1 + %arrayidx4 = getelementptr inbounds [512 x float]* @kernel2, i32 0, i32 %v.055 + %2 = load float* %arrayidx4, align 4 + %mul5 = fmul fast float %mul3, %2 + %arrayidx6 = getelementptr inbounds [512 x float]* @kernel3, i32 0, i32 %v.055 + %3 = load float* %arrayidx6, align 4 + %mul7 = fmul fast float %mul5, %3 + %arrayidx8 = getelementptr inbounds [512 x float]* @kernel4, i32 0, i32 %v.055 + %4 = load float* %arrayidx8, align 4 + %mul9 = fmul fast float %mul7, %4 + %add10 = fadd fast float %r.057, %mul9 + %arrayidx.sum = add i32 %mul, 1 + %arrayidx11 = getelementptr inbounds [1536 x float]* @src_data, i32 0, i32 %arrayidx.sum + %5 = load float* %arrayidx11, align 4 + %mul13 = fmul fast float %1, %5 + %mul15 = fmul fast float %2, %mul13 + %mul17 = fmul fast float %3, %mul15 + %mul19 = fmul fast float %4, %mul17 + %add20 = fadd fast float %g.056, %mul19 + %arrayidx.sum52 = add i32 %mul, 2 + %arrayidx21 = getelementptr inbounds [1536 x float]* @src_data, i32 0, i32 %arrayidx.sum52 + %6 = load float* %arrayidx21, align 4 + %mul23 = fmul fast float %1, %6 + %mul25 = fmul fast float %2, %mul23 + %mul27 = fmul fast float %3, %mul25 + %mul29 = fmul fast float %4, %mul27 + %add30 = fadd fast float %b.054, %mul29 + %inc = add i32 %v.055, 1 + %exitcond = icmp ne i32 %inc, %size + br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge + +for.cond.for.end_crit_edge: + %add30.lcssa = phi float [ %add30, %for.body ] + %add20.lcssa = phi float [ %add20, %for.body ] + %add10.lcssa = phi float [ %add10, %for.body ] + %phitmp = fptoui float %add10.lcssa to i8 + %phitmp60 = fptoui float %add20.lcssa to i8 + %phitmp61 = fptoui float %add30.lcssa to i8 + br label %for.end + +for.end: + %r.0.lcssa = phi i8 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ] + %g.0.lcssa = phi i8 [ %phitmp60, %for.cond.for.end_crit_edge ], [ 0, %entry ] + %b.0.lcssa = phi i8 [ %phitmp61, %for.cond.for.end_crit_edge ], [ 0, %entry ] + store i8 %r.0.lcssa, i8* @r_, align 4 + store i8 %g.0.lcssa, i8* @g_, align 4 + store i8 %b.0.lcssa, i8* @b_, align 4 + ret void +} diff --git a/test/Transforms/LoopVectorize/ARM/gcc-examples.ll b/test/Transforms/LoopVectorize/ARM/gcc-examples.ll index 6a68e81bcae0c..f2bd0ac200d4a 100644 --- a/test/Transforms/LoopVectorize/ARM/gcc-examples.ll +++ b/test/Transforms/LoopVectorize/ARM/gcc-examples.ll @@ -8,7 +8,7 @@ target triple = "thumbv7-apple-ios3.0.0" @a = common global [2048 x i32] zeroinitializer, align 16 ; Select VF = 8; -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: store <4 x i32> @@ -34,7 +34,7 @@ define void @example1() nounwind uwtable ssp { ret void } -;CHECK: @example10b +;CHECK-LABEL: @example10b( ;CHECK: load <4 x i16> ;CHECK: sext <4 x i16> ;CHECK: store <4 x i32> diff --git a/test/Transforms/LoopVectorize/ARM/lit.local.cfg b/test/Transforms/LoopVectorize/ARM/lit.local.cfg index cb77b09ef4adb..8a3ba96497e7d 100644 --- a/test/Transforms/LoopVectorize/ARM/lit.local.cfg +++ b/test/Transforms/LoopVectorize/ARM/lit.local.cfg @@ -1,5 +1,3 @@ -config.suffixes = ['.ll', '.c', '.cpp'] - targets = set(config.root.targets_to_build.split()) if not 'ARM' in targets: config.unsupported = True diff --git a/test/Transforms/LoopVectorize/ARM/width-detect.ll b/test/Transforms/LoopVectorize/ARM/width-detect.ll index c0795b6a79afa..99d7fa75ee331 100644 --- a/test/Transforms/LoopVectorize/ARM/width-detect.ll +++ b/test/Transforms/LoopVectorize/ARM/width-detect.ll @@ -3,27 +3,27 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32" target triple = "thumbv7-apple-ios3.0.0" -;CHECK:foo_F64 -;CHECK: <2 x double> +;CHECK:foo_F32 +;CHECK: <4 x float> ;CHECK:ret -define double @foo_F64(double* nocapture %A, i32 %n) nounwind uwtable readonly ssp { +define float @foo_F32(float* nocapture %A, i32 %n) nounwind uwtable readonly ssp { %1 = icmp sgt i32 %n, 0 br i1 %1, label %.lr.ph, label %._crit_edge .lr.ph: ; preds = %0, %.lr.ph %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ] - %prod.01 = phi double [ %4, %.lr.ph ], [ 0.000000e+00, %0 ] - %2 = getelementptr inbounds double* %A, i64 %indvars.iv - %3 = load double* %2, align 8 - %4 = fmul fast double %prod.01, %3 + %prod.01 = phi float [ %4, %.lr.ph ], [ 0.000000e+00, %0 ] + %2 = getelementptr inbounds float* %A, i64 %indvars.iv + %3 = load float* %2, align 8 + %4 = fmul fast float %prod.01, %3 %indvars.iv.next = add i64 %indvars.iv, 1 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 %exitcond = icmp eq i32 %lftr.wideiv, %n br i1 %exitcond, label %._crit_edge, label %.lr.ph ._crit_edge: ; preds = %.lr.ph, %0 - %prod.0.lcssa = phi double [ 0.000000e+00, %0 ], [ %4, %.lr.ph ] - ret double %prod.0.lcssa + %prod.0.lcssa = phi float [ 0.000000e+00, %0 ], [ %4, %.lr.ph ] + ret float %prod.0.lcssa } ;CHECK:foo_I8 diff --git a/test/Transforms/LoopVectorize/X86/already-vectorized.ll b/test/Transforms/LoopVectorize/X86/already-vectorized.ll new file mode 100644 index 0000000000000..885418c0fdd9e --- /dev/null +++ b/test/Transforms/LoopVectorize/X86/already-vectorized.ll @@ -0,0 +1,46 @@ +; RUN: opt < %s -debug-only=loop-vectorize -O3 -S 2>&1 | FileCheck %s +; REQUIRES: asserts +; We want to make sure that we don't even try to vectorize loops again +; The vectorizer used to mark the un-vectorized loop only as already vectorized +; thus, trying to vectorize the vectorized loop again + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@a = external global [255 x i32] + +; Function Attrs: nounwind readonly uwtable +define i32 @vect() { +; CHECK: LV: Checking a loop in "vect" +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry +; We need to make sure we did vectorize the loop +; CHECK: LV: Found a loop: for.body +; CHECK: LV: We can vectorize this loop! + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds [255 x i32]* @a, i64 0, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %add = add nsw i32 %0, %red.05 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 255 + br i1 %exitcond, label %for.end, label %for.body + +; If it did, we have two loops: +; CHECK: vector.body: +; CHECK: br {{.*}} label %vector.body, !llvm.loop [[vect:![0-9]+]] +; CHECK: for.body: +; CHECK: br {{.*}} label %for.body, !llvm.loop [[scalar:![0-9]+]] + +for.end: ; preds = %for.body + ret i32 %add +} + +; Now, we check for the Hint metadata +; CHECK: [[vect]] = metadata !{metadata [[vect]], metadata [[width:![0-9]+]], metadata [[unroll:![0-9]+]]} +; CHECK: [[width]] = metadata !{metadata !"llvm.vectorizer.width", i32 1} +; CHECK: [[unroll]] = metadata !{metadata !"llvm.vectorizer.unroll", i32 1} +; CHECK: [[scalar]] = metadata !{metadata [[scalar]], metadata [[width]], metadata [[unroll]]} + diff --git a/test/Transforms/LoopVectorize/X86/avx1.ll b/test/Transforms/LoopVectorize/X86/avx1.ll index 6c0366eae9731..01c912567b61b 100644 --- a/test/Transforms/LoopVectorize/X86/avx1.ll +++ b/test/Transforms/LoopVectorize/X86/avx1.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @read_mod_write_single_ptr +;CHECK-LABEL: @read_mod_write_single_ptr( ;CHECK: load <8 x float> ;CHECK: ret i32 define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwtable ssp { @@ -26,7 +26,7 @@ define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwta } -;CHECK: @read_mod_i64 +;CHECK-LABEL: @read_mod_i64( ;CHECK: load <2 x i64> ;CHECK: ret i32 define i32 @read_mod_i64(i64* nocapture %a, i32 %n) nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/X86/conversion-cost.ll b/test/Transforms/LoopVectorize/X86/conversion-cost.ll index 760d28deaf275..0af562db84793 100644 --- a/test/Transforms/LoopVectorize/X86/conversion-cost.ll +++ b/test/Transforms/LoopVectorize/X86/conversion-cost.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @conversion_cost1 +;CHECK-LABEL: @conversion_cost1( ;CHECK: store <32 x i8> ;CHECK: ret define i32 @conversion_cost1(i32 %n, i8* nocapture %A, float* nocapture %B) nounwind uwtable ssp { @@ -24,7 +24,7 @@ define i32 @conversion_cost1(i32 %n, i8* nocapture %A, float* nocapture %B) noun ret i32 undef } -;CHECK: @conversion_cost2 +;CHECK-LABEL: @conversion_cost2( ;CHECK: <2 x float> ;CHECK: ret define i32 @conversion_cost2(i32 %n, i8* nocapture %A, float* nocapture %B) nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/X86/cost-model.ll b/test/Transforms/LoopVectorize/X86/cost-model.ll index b7f479acf9625..98718e1e97083 100644 --- a/test/Transforms/LoopVectorize/X86/cost-model.ll +++ b/test/Transforms/LoopVectorize/X86/cost-model.ll @@ -9,7 +9,7 @@ target triple = "x86_64-apple-macosx10.8.0" @a = common global [2048 x i32] zeroinitializer, align 16 ; The program below gathers and scatters data. We better not vectorize it. -;CHECK: cost_model_1 +;CHECK-LABEL: @cost_model_1( ;CHECK-NOT: <2 x i32> ;CHECK-NOT: <4 x i32> ;CHECK-NOT: <8 x i32> diff --git a/test/Transforms/LoopVectorize/X86/gather-cost.ll b/test/Transforms/LoopVectorize/X86/gather-cost.ll new file mode 100644 index 0000000000000..09363d65eefcc --- /dev/null +++ b/test/Transforms/LoopVectorize/X86/gather-cost.ll @@ -0,0 +1,86 @@ +; RUN: opt -loop-vectorize -mtriple=x86_64-apple-macosx -S -mcpu=corei7-avx < %s | FileCheck %s +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +@kernel = global [512 x float] zeroinitializer, align 16 +@kernel2 = global [512 x float] zeroinitializer, align 16 +@kernel3 = global [512 x float] zeroinitializer, align 16 +@kernel4 = global [512 x float] zeroinitializer, align 16 +@src_data = global [1536 x float] zeroinitializer, align 16 +@r_ = global i8 0, align 1 +@g_ = global i8 0, align 1 +@b_ = global i8 0, align 1 + +; We don't want to vectorize most loops containing gathers because they are +; expensive. This function represents a point where vectorization starts to +; become beneficial. +; Make sure we are conservative and don't vectorize it. +; CHECK-NOT: x float> + +define void @_Z4testmm(i64 %size, i64 %offset) { +entry: + %cmp53 = icmp eq i64 %size, 0 + br i1 %cmp53, label %for.end, label %for.body.lr.ph + +for.body.lr.ph: + br label %for.body + +for.body: + %r.057 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add10, %for.body ] + %g.056 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add20, %for.body ] + %v.055 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ] + %b.054 = phi float [ 0.000000e+00, %for.body.lr.ph ], [ %add30, %for.body ] + %add = add i64 %v.055, %offset + %mul = mul i64 %add, 3 + %arrayidx = getelementptr inbounds [1536 x float]* @src_data, i64 0, i64 %mul + %0 = load float* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds [512 x float]* @kernel, i64 0, i64 %v.055 + %1 = load float* %arrayidx2, align 4 + %mul3 = fmul fast float %0, %1 + %arrayidx4 = getelementptr inbounds [512 x float]* @kernel2, i64 0, i64 %v.055 + %2 = load float* %arrayidx4, align 4 + %mul5 = fmul fast float %mul3, %2 + %arrayidx6 = getelementptr inbounds [512 x float]* @kernel3, i64 0, i64 %v.055 + %3 = load float* %arrayidx6, align 4 + %mul7 = fmul fast float %mul5, %3 + %arrayidx8 = getelementptr inbounds [512 x float]* @kernel4, i64 0, i64 %v.055 + %4 = load float* %arrayidx8, align 4 + %mul9 = fmul fast float %mul7, %4 + %add10 = fadd fast float %r.057, %mul9 + %arrayidx.sum = add i64 %mul, 1 + %arrayidx11 = getelementptr inbounds [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum + %5 = load float* %arrayidx11, align 4 + %mul13 = fmul fast float %1, %5 + %mul15 = fmul fast float %2, %mul13 + %mul17 = fmul fast float %3, %mul15 + %mul19 = fmul fast float %4, %mul17 + %add20 = fadd fast float %g.056, %mul19 + %arrayidx.sum52 = add i64 %mul, 2 + %arrayidx21 = getelementptr inbounds [1536 x float]* @src_data, i64 0, i64 %arrayidx.sum52 + %6 = load float* %arrayidx21, align 4 + %mul23 = fmul fast float %1, %6 + %mul25 = fmul fast float %2, %mul23 + %mul27 = fmul fast float %3, %mul25 + %mul29 = fmul fast float %4, %mul27 + %add30 = fadd fast float %b.054, %mul29 + %inc = add i64 %v.055, 1 + %exitcond = icmp ne i64 %inc, %size + br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge + +for.cond.for.end_crit_edge: + %add30.lcssa = phi float [ %add30, %for.body ] + %add20.lcssa = phi float [ %add20, %for.body ] + %add10.lcssa = phi float [ %add10, %for.body ] + %phitmp = fptoui float %add10.lcssa to i8 + %phitmp60 = fptoui float %add20.lcssa to i8 + %phitmp61 = fptoui float %add30.lcssa to i8 + br label %for.end + +for.end: + %r.0.lcssa = phi i8 [ %phitmp, %for.cond.for.end_crit_edge ], [ 0, %entry ] + %g.0.lcssa = phi i8 [ %phitmp60, %for.cond.for.end_crit_edge ], [ 0, %entry ] + %b.0.lcssa = phi i8 [ %phitmp61, %for.cond.for.end_crit_edge ], [ 0, %entry ] + store i8 %r.0.lcssa, i8* @r_, align 1 + store i8 %g.0.lcssa, i8* @g_, align 1 + store i8 %b.0.lcssa, i8* @b_, align 1 + ret void +} diff --git a/test/Transforms/LoopVectorize/X86/gcc-examples.ll b/test/Transforms/LoopVectorize/X86/gcc-examples.ll index d2d0eac305f56..e1113fdd911c0 100644 --- a/test/Transforms/LoopVectorize/X86/gcc-examples.ll +++ b/test/Transforms/LoopVectorize/X86/gcc-examples.ll @@ -9,13 +9,13 @@ target triple = "x86_64-apple-macosx10.8.0" @a = common global [2048 x i32] zeroinitializer, align 16 ; Select VF = 8; -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: store <4 x i32> ;CHECK: ret void -;UNROLL: @example1 +;UNROLL-LABEL: @example1( ;UNROLL: load <4 x i32> ;UNROLL: load <4 x i32> ;UNROLL: add nsw <4 x i32> @@ -45,12 +45,12 @@ define void @example1() nounwind uwtable ssp { } ; Select VF=4 because sext <8 x i1> to <8 x i32> is expensive. -;CHECK: @example10b +;CHECK-LABEL: @example10b( ;CHECK: load <4 x i16> ;CHECK: sext <4 x i16> ;CHECK: store <4 x i32> ;CHECK: ret void -;UNROLL: @example10b +;UNROLL-LABEL: @example10b( ;UNROLL: load <4 x i16> ;UNROLL: load <4 x i16> ;UNROLL: store <4 x i32> diff --git a/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll b/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll index 47a5e7aee4c1b..d6120e76cc0b7 100644 --- a/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll +++ b/test/Transforms/LoopVectorize/X86/illegal-parallel-loop-uniform-write.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK-NOT: <4 x i32> ;CHECK: ret void @@ -21,7 +21,7 @@ for.end.us: ; preds = %for.body3.us %indvars.iv.next34 = add i64 %indvars.iv33, 1 %lftr.wideiv35 = trunc i64 %indvars.iv.next34 to i32 %exitcond36 = icmp eq i32 %lftr.wideiv35, %m - br i1 %exitcond36, label %for.end15, label %for.body3.lr.ph.us, !llvm.loop.parallel !5 + br i1 %exitcond36, label %for.end15, label %for.body3.lr.ph.us, !llvm.loop !5 for.body3.us: ; preds = %for.body3.us, %for.body3.lr.ph.us %indvars.iv29 = phi i64 [ 0, %for.body3.lr.ph.us ], [ %indvars.iv.next30, %for.body3.us ] @@ -35,7 +35,7 @@ for.body3.us: ; preds = %for.body3.us, %for. %indvars.iv.next30 = add i64 %indvars.iv29, 1 %lftr.wideiv31 = trunc i64 %indvars.iv.next30 to i32 %exitcond32 = icmp eq i32 %lftr.wideiv31, %m - br i1 %exitcond32, label %for.end.us, label %for.body3.us, !llvm.loop.parallel !4 + br i1 %exitcond32, label %for.end.us, label %for.body3.us, !llvm.loop !4 for.body3.lr.ph.us: ; preds = %for.end.us, %entry %indvars.iv33 = phi i64 [ %indvars.iv.next34, %for.end.us ], [ 0, %entry ] diff --git a/test/Transforms/LoopVectorize/X86/lit.local.cfg b/test/Transforms/LoopVectorize/X86/lit.local.cfg index a8ad0f1a28b23..ba763cf03ffcc 100644 --- a/test/Transforms/LoopVectorize/X86/lit.local.cfg +++ b/test/Transforms/LoopVectorize/X86/lit.local.cfg @@ -1,5 +1,3 @@ -config.suffixes = ['.ll', '.c', '.cpp'] - targets = set(config.root.targets_to_build.split()) if not 'X86' in targets: config.unsupported = True diff --git a/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll b/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll index f904a8e0b1173..2c47fcb4d3890 100644 --- a/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll +++ b/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll @@ -35,7 +35,7 @@ for.body: ; preds = %for.body.for.body_c %indvars.iv.next.reload = load i64* %indvars.iv.next.reg2mem %lftr.wideiv = trunc i64 %indvars.iv.next.reload to i32 %exitcond = icmp eq i32 %lftr.wideiv, 512 - br i1 %exitcond, label %for.end, label %for.body.for.body_crit_edge, !llvm.loop.parallel !3 + br i1 %exitcond, label %for.end, label %for.body.for.body_crit_edge, !llvm.loop !3 for.body.for.body_crit_edge: ; preds = %for.body %indvars.iv.next.reload2 = load i64* %indvars.iv.next.reg2mem diff --git a/test/Transforms/LoopVectorize/X86/parallel-loops.ll b/test/Transforms/LoopVectorize/X86/parallel-loops.ll index 3f1a071e69fa8..7e156a9edad4d 100644 --- a/test/Transforms/LoopVectorize/X86/parallel-loops.ll +++ b/test/Transforms/LoopVectorize/X86/parallel-loops.ll @@ -12,7 +12,7 @@ target triple = "x86_64-unknown-linux-gnu" ; } ;} -;CHECK: @loop +;CHECK-LABEL: @loop( ;CHECK-NOT: <4 x i32> define void @loop(i32* nocapture %a, i32* nocapture %b) nounwind uwtable { entry: @@ -42,7 +42,7 @@ for.end: ; preds = %for.body ; The same loop with parallel loop metadata added to the loop branch ; and the memory instructions. -;CHECK: @parallel_loop +;CHECK-LABEL: @parallel_loop( ;CHECK: <4 x i32> define void @parallel_loop(i32* nocapture %a, i32* nocapture %b) nounwind uwtable { entry: @@ -65,7 +65,7 @@ for.body: ; preds = %for.body, %entry store i32 %2, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 %exitcond = icmp eq i32 %lftr.wideiv, 512 - br i1 %exitcond, label %for.end, label %for.body, !llvm.loop.parallel !3 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3 for.end: ; preds = %for.body ret void @@ -74,7 +74,7 @@ for.end: ; preds = %for.body ; The same loop with an illegal parallel loop metadata: the memory ; accesses refer to a different loop's identifier. -;CHECK: @mixed_metadata +;CHECK-LABEL: @mixed_metadata( ;CHECK-NOT: <4 x i32> define void @mixed_metadata(i32* nocapture %a, i32* nocapture %b) nounwind uwtable { @@ -98,7 +98,7 @@ for.body: ; preds = %for.body, %entry store i32 %2, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !6 %lftr.wideiv = trunc i64 %indvars.iv.next to i32 %exitcond = icmp eq i32 %lftr.wideiv, 512 - br i1 %exitcond, label %for.end, label %for.body, !llvm.loop.parallel !6 + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !6 for.end: ; preds = %for.body ret void diff --git a/test/Transforms/LoopVectorize/X86/rauw-bug.ll b/test/Transforms/LoopVectorize/X86/rauw-bug.ll new file mode 100644 index 0000000000000..4284fbacfa7e5 --- /dev/null +++ b/test/Transforms/LoopVectorize/X86/rauw-bug.ll @@ -0,0 +1,33 @@ +; RUN: opt -slp-vectorizer -S %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n32:64-S128" +target triple = "x86_64-apple-macosx" + +; This test used to fail under libgmalloc. Because we would try to access a +; pointer that was already deleted. +; +; llvm-lit -v --param use_gmalloc=1 --param +; gmalloc_path=/usr/lib/libgmalloc.dylib +; test/Transforms/LoopVectorize/X86/rauw-bug.ll +; +; radar://15498655 + +; CHECK: reduced +define void @reduced() { +entry: + br i1 undef, label %while.body, label %while.cond63.preheader.while.end76_crit_edge + +while.cond63.preheader.while.end76_crit_edge: + ret void + +while.body: + %d2_fx.015 = phi double [ %sub52, %while.body ], [ undef, %entry ] + %d2_fy.014 = phi double [ %sub58, %while.body ], [ undef, %entry ] + %d3_fy.013 = phi double [ %div56, %while.body ], [ undef, %entry ] + %d3_fx.012 = phi double [ %div50, %while.body ], [ undef, %entry ] + %div50 = fmul double %d3_fx.012, 1.250000e-01 + %sub52 = fsub double 0.000000e+00, %div50 + %div56 = fmul double %d3_fy.013, 1.250000e-01 + %sub58 = fsub double 0.000000e+00, %div56 + br label %while.body +} diff --git a/test/Transforms/LoopVectorize/X86/reduction-crash.ll b/test/Transforms/LoopVectorize/X86/reduction-crash.ll index f580846a0228b..3957a55414227 100644 --- a/test/Transforms/LoopVectorize/X86/reduction-crash.ll +++ b/test/Transforms/LoopVectorize/X86/reduction-crash.ll @@ -5,7 +5,7 @@ target triple = "i386-apple-darwin" ; PR15344 define void @test1(float* nocapture %arg, i32 %arg1) nounwind { -; CHECK: @test1 +; CHECK-LABEL: @test1( ; CHECK: preheader ; CHECK: insertelement <2 x double> zeroinitializer, double %tmp, i32 0 ; CHECK: vector.memcheck diff --git a/test/Transforms/LoopVectorize/X86/small-size.ll b/test/Transforms/LoopVectorize/X86/small-size.ll index f390b33c03884..14ac417bb573d 100644 --- a/test/Transforms/LoopVectorize/X86/small-size.ll +++ b/test/Transforms/LoopVectorize/X86/small-size.ll @@ -20,7 +20,7 @@ target triple = "x86_64-apple-macosx10.8.0" @dj = common global [1024 x i32] zeroinitializer, align 16 ; We can optimize this test without a tail. -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: store <4 x i32> @@ -47,7 +47,7 @@ define void @example1() optsize { } ; Can't vectorize in 'optsize' mode because we need a tail. -;CHECK: @example2 +;CHECK-LABEL: @example2( ;CHECK-NOT: store <4 x i32> ;CHECK: ret void define void @example2(i32 %n, i32 %x) optsize { @@ -92,7 +92,7 @@ define void @example2(i32 %n, i32 %x) optsize { } ; N is unknown, we need a tail. Can't vectorize. -;CHECK: @example3 +;CHECK-LABEL: @example3( ;CHECK-NOT: <4 x i32> ;CHECK: ret void define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture %q) optsize { @@ -117,7 +117,7 @@ define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture ; We can't vectorize this one because we need a runtime ptr check. -;CHECK: @example23 +;CHECK-LABEL: @example23( ;CHECK-NOT: <4 x i32> ;CHECK: ret void define void @example23(i16* nocapture %src, i32* nocapture %dst) optsize { @@ -143,7 +143,7 @@ define void @example23(i16* nocapture %src, i32* nocapture %dst) optsize { ; We CAN vectorize this example because the pointers are marked as noalias. -;CHECK: @example23b +;CHECK-LABEL: @example23b( ;CHECK: <4 x i32> ;CHECK: ret void define void @example23b(i16* noalias nocapture %src, i32* noalias nocapture %dst) optsize { diff --git a/test/Transforms/LoopVectorize/X86/tripcount.ll b/test/Transforms/LoopVectorize/X86/tripcount.ll new file mode 100644 index 0000000000000..6b38bacf88884 --- /dev/null +++ b/test/Transforms/LoopVectorize/X86/tripcount.ll @@ -0,0 +1,39 @@ +; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-unroll=1 -mcpu=prescott < %s | FileCheck %s + +target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128" +target triple = "i386-unknown-freebsd11.0" + +@big = external global [0 x i32] + +; PR18049 +; We need to truncate the exit count to i32. This is legal because the +; arithmetic is signed (%inc is nsw). + +; CHECK-LABEL: tripcount +; CHECK: trunc i64 %count to i32 + +define void @tripcount(i64 %count) { +entry: + %cmp6 = icmp sgt i64 %count, 0 + br i1 %cmp6, label %for.body.preheader, label %for.end + +for.body.preheader: + br label %for.body + +for.body: + %i.07 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ] + %arrayidx = getelementptr inbounds [0 x i32]* @big, i32 0, i32 %i.07 + %0 = load i32* %arrayidx, align 4 + %neg = xor i32 %0, -1 + store i32 %neg, i32* %arrayidx, align 4 + %inc = add nsw i32 %i.07, 1 + %conv = sext i32 %inc to i64 + %cmp = icmp slt i64 %conv, %count + br i1 %cmp, label %for.body, label %for.end.loopexit + +for.end.loopexit: + br label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/X86/unroll-pm.ll b/test/Transforms/LoopVectorize/X86/unroll-pm.ll new file mode 100644 index 0000000000000..5064fec286ce0 --- /dev/null +++ b/test/Transforms/LoopVectorize/X86/unroll-pm.ll @@ -0,0 +1,31 @@ +; RUN: opt < %s -O2 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -S | FileCheck %s +; RUN: opt < %s -O2 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx -force-vector-width=4 -disable-loop-unrolling -S | FileCheck %s -check-prefix=CHECK-NOUNRL + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" +;CHECK-LABEL: @bar( +;CHECK: store <4 x i32> +;CHECK: store <4 x i32> +;CHECK: ret +;CHECK-NOUNRL-LABEL: @bar( +;CHECK-NOUNRL: store <4 x i32> +;CHECK-NOUNRL-NOT: store <4 x i32> +;CHECK-NOUNRL: ret +define i32 @bar(i32* nocapture %A, i32 %n) nounwind uwtable ssp { + %1 = icmp sgt i32 %n, 0 + br i1 %1, label %.lr.ph, label %._crit_edge + +.lr.ph: ; preds = %0, %.lr.ph + %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ] + %2 = getelementptr inbounds i32* %A, i64 %indvars.iv + %3 = load i32* %2, align 4 + %4 = add nsw i32 %3, 6 + store i32 %4, i32* %2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %._crit_edge, label %.lr.ph + +._crit_edge: ; preds = %.lr.ph, %0 + ret i32 undef +} diff --git a/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll b/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll index ef63a145d0c1e..ea107dc4dc51e 100644 --- a/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll +++ b/test/Transforms/LoopVectorize/X86/unroll-small-loops.ll @@ -2,7 +2,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK: load <4 x i32> ;CHECK-NOT: load <4 x i32> ;CHECK: store <4 x i32> @@ -26,7 +26,7 @@ define i32 @foo(i32* nocapture %A) nounwind uwtable ssp { ret i32 undef } -;CHECK: @bar +;CHECK-LABEL: @bar( ;CHECK: store <4 x i32> ;CHECK: store <4 x i32> ;CHECK: ret diff --git a/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll b/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll index b66119f4ef59e..efc93d94a7c51 100644 --- a/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll +++ b/test/Transforms/LoopVectorize/X86/x86_fp80-vector-store.ll @@ -5,7 +5,7 @@ target triple = "x86_64-apple-macosx10.7.0" @x = common global [1024 x x86_fp80] zeroinitializer, align 16 -;CHECK: @example +;CHECK-LABEL: @example( ;CHECK-NOT: bitcast x86_fp80* {{%[^ ]+}} to <{{[2-9][0-9]*}} x x86_fp80>* ;CHECK: store ;CHECK: ret void diff --git a/test/Transforms/LoopVectorize/XCore/lit.local.cfg b/test/Transforms/LoopVectorize/XCore/lit.local.cfg new file mode 100644 index 0000000000000..4d17d46420455 --- /dev/null +++ b/test/Transforms/LoopVectorize/XCore/lit.local.cfg @@ -0,0 +1,3 @@ +targets = set(config.root.targets_to_build.split()) +if not 'XCore' in targets: + config.unsupported = True diff --git a/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll b/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll new file mode 100644 index 0000000000000..a099daa740e56 --- /dev/null +++ b/test/Transforms/LoopVectorize/XCore/no-vector-registers.ll @@ -0,0 +1,23 @@ +; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-unroll=2 -S -mtriple=xcore | FileCheck %s + +target datalayout = "e-p:32:32:32-a0:0:32-n32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f16:16:32-f32:32:32-f64:32:32" +target triple = "xcore" +; The xcore target has no vector registers, so loop should not be vectorized. +;CHECK-LABEL: @f( +;CHECK: entry: +;CHECK-NOT: vector.body +;CHECK-NEXT: br label %do.body +define void @f(i8* nocapture %ptr, i32 %len) { +entry: + br label %do.body +do.body: + %ptr.addr.0 = phi i8* [ %ptr, %entry ], [ %incdec.ptr, %do.body ] + %len.addr.0 = phi i32 [ %len, %entry ], [ %dec, %do.body ] + %incdec.ptr = getelementptr inbounds i8* %ptr.addr.0, i32 1 + store i8 0, i8* %ptr.addr.0, align 1 + %dec = add nsw i32 %len.addr.0, -1 + %tobool = icmp eq i32 %len.addr.0, 0 + br i1 %tobool, label %do.end, label %do.body +do.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/align.ll b/test/Transforms/LoopVectorize/align.ll new file mode 100644 index 0000000000000..84b03615d374c --- /dev/null +++ b/test/Transforms/LoopVectorize/align.ll @@ -0,0 +1,33 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +; Make sure we output the abi alignment if no alignment is specified. + +;CHECK-LABEL: @align +;CHECK: load <4 x i32>* {{.*}} align 4 +;CHECK: load <4 x i32>* {{.*}} align 4 +;CHECK: store <4 x i32> {{.*}} align 4 + +define void @align(i32* %a, i32* %b, i32* %c) nounwind uwtable ssp { + br label %1 + +; <label>:1 ; preds = %1, %0 + %indvars.iv = phi i64 [ 0, %0 ], [ %indvars.iv.next, %1 ] + %2 = getelementptr inbounds i32* %b, i64 %indvars.iv + %3 = load i32* %2 + %4 = getelementptr inbounds i32* %c, i64 %indvars.iv + %5 = load i32* %4 + %6 = add nsw i32 %5, %3 + %7 = getelementptr inbounds i32* %a, i64 %indvars.iv + store i32 %6, i32* %7 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, 128 + br i1 %exitcond, label %8, label %1 + +; <label>:8 ; preds = %1 + ret void +} + diff --git a/test/Transforms/LoopVectorize/bsd_regex.ll b/test/Transforms/LoopVectorize/bsd_regex.ll index a14b92d229b45..7b712729a1cd6 100644 --- a/test/Transforms/LoopVectorize/bsd_regex.ll +++ b/test/Transforms/LoopVectorize/bsd_regex.ll @@ -4,7 +4,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ;PR 15830. -;CHECK: foo +;CHECK-LABEL: @foo( ; When scalarizing stores we need to preserve the original order. ; Make sure that we are extracting in the correct order (0101, and not 0011). ;CHECK: extractelement <2 x i64> {{.*}}, i32 0 diff --git a/test/Transforms/LoopVectorize/cast-induction.ll b/test/Transforms/LoopVectorize/cast-induction.ll index 2aa29ed2c820a..255ce9c77eafc 100644 --- a/test/Transforms/LoopVectorize/cast-induction.ll +++ b/test/Transforms/LoopVectorize/cast-induction.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0" @a = common global [2048 x i32] zeroinitializer, align 16 -;CHECK: @example12 +;CHECK-LABEL: @example12( ;CHECK: trunc i64 ;CHECK: store <4 x i32> ;CHECK: ret void diff --git a/test/Transforms/LoopVectorize/cpp-new-array.ll b/test/Transforms/LoopVectorize/cpp-new-array.ll index da0fb05fe8434..c8215a107de7a 100644 --- a/test/Transforms/LoopVectorize/cpp-new-array.ll +++ b/test/Transforms/LoopVectorize/cpp-new-array.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @cpp_new_arrays +;CHECK-LABEL: @cpp_new_arrays( ;CHECK: sext i32 ;CHECK: load <4 x float> ;CHECK: fadd <4 x float> diff --git a/test/Transforms/LoopVectorize/dbg.value.ll b/test/Transforms/LoopVectorize/dbg.value.ll index 127d479b3a060..2497b25ea1da9 100644 --- a/test/Transforms/LoopVectorize/dbg.value.ll +++ b/test/Transforms/LoopVectorize/dbg.value.ll @@ -8,7 +8,7 @@ target triple = "x86_64-apple-macosx10.8.0" @B = global [1024 x i32] zeroinitializer, align 16 @C = global [1024 x i32] zeroinitializer, align 16 -; CHECK: @test +; CHECK-LABEL: @test( define i32 @test() #0 { entry: tail call void @llvm.dbg.value(metadata !1, i64 0, metadata !9), !dbg !18 @@ -38,30 +38,33 @@ declare void @llvm.dbg.declare(metadata, metadata) #1 declare void @llvm.dbg.value(metadata, i64, metadata) #1 -attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" } +attributes #0 = { nounwind ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "relocation-model"="pic" "ssp-buffers-size"="8" } attributes #1 = { nounwind readnone } !llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!26} -!0 = metadata !{i32 786449, i32 0, i32 4, metadata !"test", metadata !"/path/to/somewhere", metadata !"clang", i1 true, i1 true, metadata !"", i32 0, metadata !1, metadata !1, metadata !2, metadata !11, metadata !""} +!0 = metadata !{i32 786449, metadata !25, i32 4, metadata !"clang", i1 true, metadata !"", i32 0, metadata !1, metadata !1, metadata !2, metadata !11, null, metadata !""} !1 = metadata !{i32 0} !2 = metadata !{metadata !3} -!3 = metadata !{i32 786478, i32 0, metadata !4, metadata !"test", metadata !"test", metadata !"test", metadata !4, i32 5, metadata !5, i1 false, i1 true, i32 0, i32 0, null, i32 256, i1 true, i32 ()* @test, null, null, metadata !8, i32 5} -!4 = metadata !{i32 786473, metadata !"test", metadata !"/path/to/somewhere", null} -!5 = metadata !{i32 786453, i32 0, metadata !"", i32 0, i32 0, i64 0, i64 0, i64 0, i32 0, null, metadata !6, i32 0, i32 0} +!3 = metadata !{i32 786478, metadata !25, metadata !4, metadata !"test", metadata !"test", metadata !"test", i32 5, metadata !5, i1 false, i1 true, i32 0, i32 0, null, i32 256, i1 true, i32 ()* @test, null, null, metadata !8, i32 5} +!4 = metadata !{i32 786473, metadata !25} +!5 = metadata !{i32 786453, i32 0, null, i32 0, i32 0, i64 0, i64 0, i64 0, i32 0, null, metadata !6, i32 0, null, null, null} ; [ DW_TAG_subroutine_type ] [line 0, size 0, align 0, offset 0] [from ] !6 = metadata !{metadata !7} -!7 = metadata !{i32 786468, null, metadata !"int", null, i32 0, i64 32, i64 32, i64 0, i32 0, i32 5} +!7 = metadata !{i32 786468, null, null, metadata !"int", i32 0, i64 32, i64 32, i64 0, i32 0, i32 5} !8 = metadata !{metadata !9} !9 = metadata !{i32 786688, metadata !10, metadata !"i", metadata !4, i32 6, metadata !7, i32 0, i32 0} -!10 = metadata !{i32 786443, metadata !3, i32 6, i32 0, metadata !4, i32 0} +!10 = metadata !{i32 786443, metadata !25, metadata !3, i32 6, i32 0, i32 0} !11 = metadata !{metadata !12, metadata !16, metadata !17} !12 = metadata !{i32 786484, i32 0, null, metadata !"A", metadata !"A", metadata !"", metadata !4, i32 1, metadata !13, i32 0, i32 1, [1024 x i32]* @A, null} -!13 = metadata !{i32 786433, null, metadata !"", null, i32 0, i64 32768, i64 32, i32 0, i32 0, metadata !7, metadata !14, i32 0, i32 0} +!13 = metadata !{i32 786433, null, null, null, i32 0, i64 32768, i64 32, i32 0, i32 0, metadata !7, metadata !14, i32 0, null, null, null} ; [ DW_TAG_array_type ] [line 0, size 32768, align 32, offset 0] [from int] !14 = metadata !{metadata !15} !15 = metadata !{i32 786465, i64 0, i64 1024} !16 = metadata !{i32 786484, i32 0, null, metadata !"B", metadata !"B", metadata !"", metadata !4, i32 2, metadata !13, i32 0, i32 1, [1024 x i32]* @B, null} !17 = metadata !{i32 786484, i32 0, null, metadata !"C", metadata !"C", metadata !"", metadata !4, i32 3, metadata !13, i32 0, i32 1, [1024 x i32]* @C, null} !18 = metadata !{i32 6, i32 0, metadata !10, null} !19 = metadata !{i32 7, i32 0, metadata !20, null} -!20 = metadata !{i32 786443, metadata !10, i32 6, i32 0, metadata !4, i32 1} +!20 = metadata !{i32 786443, metadata !25, metadata !10, i32 6, i32 0, i32 1} !24 = metadata !{i32 9, i32 0, metadata !3, null} +!25 = metadata !{metadata !"test", metadata !"/path/to/somewhere"} +!26 = metadata !{i32 1, metadata !"Debug Info Version", i32 1} diff --git a/test/Transforms/LoopVectorize/debugloc.ll b/test/Transforms/LoopVectorize/debugloc.ll new file mode 100644 index 0000000000000..bf0b4184b7a13 --- /dev/null +++ b/test/Transforms/LoopVectorize/debugloc.ll @@ -0,0 +1,90 @@ +; RUN: opt -S < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=2 | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; Make sure we are preserving debug info in the vectorized code. + +; CHECK: for.body.lr.ph +; CHECK: cmp.zero = icmp eq i64 {{.*}}, 0, !dbg ![[LOC:[0-9]+]] +; CHECK: vector.body +; CHECK: index {{.*}}, !dbg ![[LOC]] +; CHECK: getelementptr inbounds i32* %a, {{.*}}, !dbg ![[LOC2:[0-9]+]] +; CHECK: load <2 x i32>* {{.*}}, !dbg ![[LOC2]] +; CHECK: add <2 x i32> {{.*}}, !dbg ![[LOC2]] +; CHECK: add i64 %index, 2, !dbg ![[LOC]] +; CHECK: icmp eq i64 %index.next, %end.idx.rnd.down, !dbg ![[LOC]] +; CHECK: middle.block +; CHECK: add <2 x i32> %rdx.vec.exit.phi, %rdx.shuf, !dbg ![[LOC2]] +; CHECK: extractelement <2 x i32> %bin.rdx, i32 0, !dbg ![[LOC2]] + +define i32 @f(i32* nocapture %a, i32 %size) #0 { +entry: + tail call void @llvm.dbg.value(metadata !{i32* %a}, i64 0, metadata !13), !dbg !19 + tail call void @llvm.dbg.value(metadata !{i32 %size}, i64 0, metadata !14), !dbg !19 + tail call void @llvm.dbg.value(metadata !2, i64 0, metadata !15), !dbg !20 + tail call void @llvm.dbg.value(metadata !2, i64 0, metadata !16), !dbg !21 + %cmp4 = icmp eq i32 %size, 0, !dbg !21 + br i1 %cmp4, label %for.end, label %for.body.lr.ph, !dbg !21 + +for.body.lr.ph: ; preds = %entry + br label %for.body, !dbg !21 + +for.body: ; preds = %for.body.lr.ph, %for.body + %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] + %sum.05 = phi i32 [ 0, %for.body.lr.ph ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv, !dbg !22 + %0 = load i32* %arrayidx, align 4, !dbg !22 + %add = add i32 %0, %sum.05, !dbg !22 + tail call void @llvm.dbg.value(metadata !{i32 %add.lcssa}, i64 0, metadata !15), !dbg !22 + %indvars.iv.next = add i64 %indvars.iv, 1, !dbg !21 + tail call void @llvm.dbg.value(metadata !{null}, i64 0, metadata !16), !dbg !21 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !21 + %exitcond = icmp ne i32 %lftr.wideiv, %size, !dbg !21 + br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge, !dbg !21 + +for.cond.for.end_crit_edge: ; preds = %for.body + %add.lcssa = phi i32 [ %add, %for.body ] + br label %for.end, !dbg !21 + +for.end: ; preds = %entry, %for.cond.for.end_crit_edge + %sum.0.lcssa = phi i32 [ %add.lcssa, %for.cond.for.end_crit_edge ], [ 0, %entry ] + ret i32 %sum.0.lcssa, !dbg !26 +} + +; Function Attrs: nounwind readnone +declare void @llvm.dbg.declare(metadata, metadata) #1 + +; Function Attrs: nounwind readnone +declare void @llvm.dbg.value(metadata, i64, metadata) #1 + +attributes #0 = { nounwind readonly ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!18, !27} + +!0 = metadata !{i32 786449, metadata !1, i32 12, metadata !"clang version 3.4 (trunk 185038) (llvm/trunk 185097)", i1 true, metadata !"", i32 0, metadata !2, metadata !2, metadata !3, metadata !2, metadata !2, metadata !""} ; [ DW_TAG_compile_unit ] [/Volumes/Data/backedup/dev/os/llvm/debug/-] [DW_LANG_C99] +!1 = metadata !{metadata !"-", metadata !"/Volumes/Data/backedup/dev/os/llvm/debug"} +!2 = metadata !{i32 0} +!3 = metadata !{metadata !4} +!4 = metadata !{i32 786478, metadata !5, metadata !6, metadata !"f", metadata !"f", metadata !"", i32 3, metadata !7, i1 false, i1 true, i32 0, i32 0, null, i32 256, i1 true, i32 (i32*, i32)* @f, null, null, metadata !12, i32 3} ; [ DW_TAG_subprogram ] [line 3] [def] [f] +!5 = metadata !{metadata !"<stdin>", metadata !"/Volumes/Data/backedup/dev/os/llvm/debug"} +!6 = metadata !{i32 786473, metadata !5} ; [ DW_TAG_file_type ] [/Volumes/Data/backedup/dev/os/llvm/debug/<stdin>] +!7 = metadata !{i32 786453, i32 0, null, metadata !"", i32 0, i64 0, i64 0, i64 0, i32 0, null, metadata !8, i32 0, null, null, null} ; [ DW_TAG_subroutine_type ] [line 0, size 0, align 0, offset 0] [from ] +!8 = metadata !{metadata !9, metadata !10, metadata !11} +!9 = metadata !{i32 786468, null, null, metadata !"int", i32 0, i64 32, i64 32, i64 0, i32 0, i32 5} ; [ DW_TAG_base_type ] [int] [line 0, size 32, align 32, offset 0, enc DW_ATE_signed] +!10 = metadata !{i32 786447, null, null, metadata !"", i32 0, i64 64, i64 64, i64 0, i32 0, metadata !9} ; [ DW_TAG_pointer_type ] [line 0, size 64, align 64, offset 0] [from int] +!11 = metadata !{i32 786468, null, null, metadata !"unsigned int", i32 0, i64 32, i64 32, i64 0, i32 0, i32 7} ; [ DW_TAG_base_type ] [unsigned int] [line 0, size 32, align 32, offset 0, enc DW_ATE_unsigned] +!12 = metadata !{metadata !13, metadata !14, metadata !15, metadata !16} +!13 = metadata !{i32 786689, metadata !4, metadata !"a", metadata !6, i32 16777219, metadata !10, i32 0, i32 0} ; [ DW_TAG_arg_variable ] [a] [line 3] +!14 = metadata !{i32 786689, metadata !4, metadata !"size", metadata !6, i32 33554435, metadata !11, i32 0, i32 0} ; [ DW_TAG_arg_variable ] [size] [line 3] +!15 = metadata !{i32 786688, metadata !4, metadata !"sum", metadata !6, i32 4, metadata !11, i32 0, i32 0} ; [ DW_TAG_auto_variable ] [sum] [line 4] +!16 = metadata !{i32 786688, metadata !17, metadata !"i", metadata !6, i32 5, metadata !11, i32 0, i32 0} ; [ DW_TAG_auto_variable ] [i] [line 5] +!17 = metadata !{i32 786443, metadata !5, metadata !4, i32 5, i32 0, i32 0} ; [ DW_TAG_lexical_block ] [/Volumes/Data/backedup/dev/os/llvm/debug/<stdin>] +!18 = metadata !{i32 2, metadata !"Dwarf Version", i32 3} +!19 = metadata !{i32 3, i32 0, metadata !4, null} +!20 = metadata !{i32 4, i32 0, metadata !4, null} +!21 = metadata !{i32 5, i32 0, metadata !17, null} +!22 = metadata !{i32 6, i32 0, metadata !17, null} +!26 = metadata !{i32 7, i32 0, metadata !4, null} +!27 = metadata !{i32 1, metadata !"Debug Info Version", i32 1} diff --git a/test/Transforms/LoopVectorize/ee-crash.ll b/test/Transforms/LoopVectorize/ee-crash.ll new file mode 100644 index 0000000000000..8a4f8ce3c122d --- /dev/null +++ b/test/Transforms/LoopVectorize/ee-crash.ll @@ -0,0 +1,35 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +; This test checks that we deal with an in-loop extractelement (for now, this +; means not crashing by not vectorizing). +; CHECK-LABEL: @_Z4foo1Pii( +; CHECK-NOT: <4 x i32> +; CHECK: ret +define i32 @_Z4foo1Pii(i32* %A, i32 %n, <2 x i32> %q) #0 { +entry: + %idx.ext = sext i32 %n to i64 + %add.ptr = getelementptr inbounds i32* %A, i64 %idx.ext + %cmp3.i = icmp eq i32 %n, 0 + br i1 %cmp3.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i + +for.body.i: ; preds = %entry, %for.body.i + %__init.addr.05.i = phi i32 [ %add.i, %for.body.i ], [ 0, %entry ] + %__first.addr.04.i = phi i32* [ %incdec.ptr.i, %for.body.i ], [ %A, %entry ] + %0 = load i32* %__first.addr.04.i, align 4 + %q1 = extractelement <2 x i32> %q, i32 %n + %q2 = add nsw i32 %0, %q1 + %add.i = add nsw i32 %q2, %__init.addr.05.i + %incdec.ptr.i = getelementptr inbounds i32* %__first.addr.04.i, i64 1 + %cmp.i = icmp eq i32* %incdec.ptr.i, %add.ptr + br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i + +_ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %for.body.i, %entry + %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ] + ret i32 %__init.addr.0.lcssa.i +} + +attributes #0 = { nounwind readonly ssp uwtable } + diff --git a/test/Transforms/LoopVectorize/flags.ll b/test/Transforms/LoopVectorize/flags.ll index 656912e178f91..a4ebb42848812 100644 --- a/test/Transforms/LoopVectorize/flags.ll +++ b/test/Transforms/LoopVectorize/flags.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @flags1 +;CHECK-LABEL: @flags1( ;CHECK: load <4 x i32> ;CHECK: mul nsw <4 x i32> ;CHECK: store <4 x i32> @@ -28,7 +28,7 @@ define i32 @flags1(i32 %n, i32* nocapture %A) nounwind uwtable ssp { } -;CHECK: @flags2 +;CHECK-LABEL: @flags2( ;CHECK: load <4 x i32> ;CHECK: mul <4 x i32> ;CHECK: store <4 x i32> diff --git a/test/Transforms/LoopVectorize/float-reduction.ll b/test/Transforms/LoopVectorize/float-reduction.ll index 54ca172e86965..c45098dd2c3b9 100644 --- a/test/Transforms/LoopVectorize/float-reduction.ll +++ b/test/Transforms/LoopVectorize/float-reduction.ll @@ -2,7 +2,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK: fadd <4 x float> ;CHECK: ret define float @foo(float* nocapture %A, i32* nocapture %n) nounwind uwtable readonly ssp { diff --git a/test/Transforms/LoopVectorize/funcall.ll b/test/Transforms/LoopVectorize/funcall.ll new file mode 100644 index 0000000000000..f1f068c43db30 --- /dev/null +++ b/test/Transforms/LoopVectorize/funcall.ll @@ -0,0 +1,32 @@ +; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-unroll=1 < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; Make sure we can vectorize loops with functions to math library functions. +; They might read the rounding mode but we are only vectorizing loops that +; contain a limited set of function calls and none of them sets the rounding +; mode, so vectorizing them is safe. + +; CHECK-LABEL: @test( +; CHECK: <2 x double> + +define void @test(double* %d, double %t) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds double* %d, i64 %indvars.iv + %0 = load double* %arrayidx, align 8 + %1 = tail call double @llvm.pow.f64(double %0, double %t) + store double %1, double* %arrayidx, align 8 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +declare double @llvm.pow.f64(double, double) diff --git a/test/Transforms/LoopVectorize/gcc-examples.ll b/test/Transforms/LoopVectorize/gcc-examples.ll index f335557c0019a..d8959d4c106a7 100644 --- a/test/Transforms/LoopVectorize/gcc-examples.ll +++ b/test/Transforms/LoopVectorize/gcc-examples.ll @@ -20,12 +20,12 @@ target triple = "x86_64-apple-macosx10.8.0" @dd = common global [1024 x float] zeroinitializer, align 16 @dj = common global [1024 x i32] zeroinitializer, align 16 -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: store <4 x i32> ;CHECK: ret void -;UNROLL: @example1 +;UNROLL-LABEL: @example1( ;UNROLL: load <4 x i32> ;UNROLL: load <4 x i32> ;UNROLL: load <4 x i32> @@ -60,10 +60,10 @@ define void @example1() nounwind uwtable ssp { ret void } -;CHECK: @example2 +;CHECK-LABEL: @example2( ;CHECK: store <4 x i32> ;CHECK: ret void -;UNROLL: @example2 +;UNROLL-LABEL: @example2( ;UNROLL: store <4 x i32> ;UNROLL: store <4 x i32> ;UNROLL: store <4 x i32> @@ -110,10 +110,10 @@ define void @example2(i32 %n, i32 %x) nounwind uwtable ssp { ret void } -;CHECK: @example3 +;CHECK-LABEL: @example3( ;CHECK: <4 x i32> ;CHECK: ret void -;UNROLL: @example3 +;UNROLL-LABEL: @example3( ;UNROLL: <4 x i32> ;UNROLL: <4 x i32> ;UNROLL: <4 x i32> @@ -139,10 +139,10 @@ define void @example3(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture ret void } -;CHECK: @example4 +;CHECK-LABEL: @example4( ;CHECK: load <4 x i32> ;CHECK: ret void -;UNROLL: @example4 +;UNROLL-LABEL: @example4( ;UNROLL: load <4 x i32> ;UNROLL: load <4 x i32> ;UNROLL: load <4 x i32> @@ -205,10 +205,10 @@ define void @example4(i32 %n, i32* noalias nocapture %p, i32* noalias nocapture ret void } -;CHECK: @example8 +;CHECK-LABEL: @example8( ;CHECK: store <4 x i32> ;CHECK: ret void -;UNROLL: @example8 +;UNROLL-LABEL: @example8( ;UNROLL: store <4 x i32> ;UNROLL: store <4 x i32> ;UNROLL: store <4 x i32> @@ -240,7 +240,7 @@ define void @example8(i32 %x) nounwind uwtable ssp { ret void } -;CHECK: @example9 +;CHECK-LABEL: @example9( ;CHECK: phi <4 x i32> ;CHECK: ret i32 define i32 @example9() nounwind uwtable readonly ssp { @@ -264,7 +264,7 @@ define i32 @example9() nounwind uwtable readonly ssp { ret i32 %7 } -;CHECK: @example10a +;CHECK-LABEL: @example10a( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: load <4 x i16> @@ -299,7 +299,7 @@ define void @example10a(i16* noalias nocapture %sa, i16* noalias nocapture %sb, ret void } -;CHECK: @example10b +;CHECK-LABEL: @example10b( ;CHECK: load <4 x i16> ;CHECK: sext <4 x i16> ;CHECK: store <4 x i32> @@ -323,7 +323,7 @@ define void @example10b(i16* noalias nocapture %sa, i16* noalias nocapture %sb, ret void } -;CHECK: @example11 +;CHECK-LABEL: @example11( ;CHECK: load i32 ;CHECK: load i32 ;CHECK: load i32 @@ -367,7 +367,7 @@ define void @example11() nounwind uwtable ssp { ret void } -;CHECK: @example12 +;CHECK-LABEL: @example12( ;CHECK: trunc i64 ;CHECK: store <4 x i32> ;CHECK: ret void @@ -389,7 +389,7 @@ define void @example12() nounwind uwtable ssp { } ; Can't vectorize because of reductions. -;CHECK: @example13 +;CHECK-LABEL: @example13( ;CHECK-NOT: <4 x i32> ;CHECK: ret void define void @example13(i32** nocapture %A, i32** nocapture %B, i32* nocapture %out) nounwind uwtable ssp { @@ -430,7 +430,7 @@ define void @example13(i32** nocapture %A, i32** nocapture %B, i32* nocapture %o } ; Can vectorize. -;CHECK: @example14 +;CHECK-LABEL: @example14( ;CHECK: <4 x i32> ;CHECK: ret void define void @example14(i32** nocapture %in, i32** nocapture %coeff, i32* nocapture %out) nounwind uwtable ssp { @@ -575,7 +575,7 @@ define void @example14(i32** nocapture %in, i32** nocapture %coeff, i32* nocaptu ret void } -;CHECK: @example21 +;CHECK-LABEL: @example21( ;CHECK: load <4 x i32> ;CHECK: shufflevector {{.*}} <i32 3, i32 2, i32 1, i32 0> ;CHECK: ret i32 @@ -603,7 +603,7 @@ define i32 @example21(i32* nocapture %b, i32 %n) nounwind uwtable readonly ssp { ret i32 %a.0.lcssa } -;CHECK: @example23 +;CHECK-LABEL: @example23( ;CHECK: <4 x i32> ;CHECK: ret void define void @example23(i16* nocapture %src, i32* nocapture %dst) nounwind uwtable ssp { @@ -627,7 +627,7 @@ define void @example23(i16* nocapture %src, i32* nocapture %dst) nounwind uwtabl ret void } -;CHECK: @example24 +;CHECK-LABEL: @example24( ;CHECK: shufflevector <4 x i16> ;CHECK: ret void define void @example24(i16 signext %x, i16 signext %y) nounwind uwtable ssp { @@ -653,7 +653,7 @@ define void @example24(i16 signext %x, i16 signext %y) nounwind uwtable ssp { ret void } -;CHECK: @example25 +;CHECK-LABEL: @example25( ;CHECK: and <4 x i1> ;CHECK: zext <4 x i1> ;CHECK: ret void diff --git a/test/Transforms/LoopVectorize/global_alias.ll b/test/Transforms/LoopVectorize/global_alias.ll index 121da8ba7e16c..0118fb47412a4 100644 --- a/test/Transforms/LoopVectorize/global_alias.ll +++ b/test/Transforms/LoopVectorize/global_alias.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -O3 -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s +; RUN: opt < %s -O1 -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:64:128-a0:0:64-n32-S64" @@ -22,7 +22,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; Foo.A[i] = Foo.B[i] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias01 +; CHECK-LABEL: define i32 @noAlias01( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -70,7 +70,7 @@ for.end: ; preds = %for.cond ; Foo.A[i] = Foo.B[i+10] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias02 +; CHECK-LABEL: define i32 @noAlias02( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -119,7 +119,7 @@ for.end: ; preds = %for.cond ; Foo.A[i+10] = Foo.B[i] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias03 +; CHECK-LABEL: define i32 @noAlias03( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -168,7 +168,7 @@ for.end: ; preds = %for.cond ; *(PA+i) = *(PB+i) + a; ; return *(PA+a); ; } -; CHECK: define i32 @noAlias04 +; CHECK-LABEL: define i32 @noAlias04( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret ; @@ -222,7 +222,7 @@ for.end: ; preds = %for.cond ; Bar.A[N][i] = Bar.B[N][i] + a; ; return Bar.A[N][a]; ; } -; CHECK: define i32 @noAlias05 +; CHECK-LABEL: define i32 @noAlias05( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -278,7 +278,7 @@ for.end: ; preds = %for.cond ; Bar.A[N][i] = Bar.A[N+1][i] + a; ; return Bar.A[N][a]; ; } -; CHECK: define i32 @noAlias06 +; CHECK-LABEL: define i32 @noAlias06( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -335,10 +335,9 @@ for.end: ; preds = %for.cond ; Foo.A[SIZE-i-1] = Foo.B[SIZE-i-1] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias07 -; CHECK: sub nsw <4 x i32> +; CHECK-LABEL: define i32 @noAlias07( +; CHECK: store <4 x i32> ; CHECK: ret - define i32 @noAlias07(i32 %a) #0 { entry: %a.addr = alloca i32, align 4 @@ -387,7 +386,7 @@ for.end: ; preds = %for.cond ; Foo.A[SIZE-i-1] = Foo.B[SIZE-i-10] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias08 +; CHECK-LABEL: define i32 @noAlias08( ; CHECK: sub nsw <4 x i32> ; CHECK: ret @@ -439,7 +438,7 @@ for.end: ; preds = %for.cond ; Foo.A[SIZE-i-10] = Foo.B[SIZE-i-1] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias09 +; CHECK-LABEL: define i32 @noAlias09( ; CHECK: sub nsw <4 x i32> ; CHECK: ret @@ -491,7 +490,7 @@ for.end: ; preds = %for.cond ; *(PA+SIZE-i-1) = *(PB+SIZE-i-1) + a; ; return *(PA+a); ; } -; CHECK: define i32 @noAlias10 +; CHECK-LABEL: define i32 @noAlias10( ; CHECK-NOT: sub nsw <4 x i32> ; CHECK: ret ; @@ -551,8 +550,8 @@ for.end: ; preds = %for.cond ; Bar.A[N][SIZE-i-1] = Bar.B[N][SIZE-i-1] + a; ; return Bar.A[N][a]; ; } -; CHECK: define i32 @noAlias11 -; CHECK: sub nsw <4 x i32> +; CHECK-LABEL: define i32 @noAlias11( +; CHECK: store <4 x i32> ; CHECK: ret define i32 @noAlias11(i32 %a) #0 { @@ -611,8 +610,8 @@ for.end: ; preds = %for.cond ; Bar.A[N][SIZE-i-1] = Bar.A[N+1][SIZE-i-1] + a; ; return Bar.A[N][a]; ; } -; CHECK: define i32 @noAlias12 -; CHECK: sub nsw <4 x i32> +; CHECK-LABEL: define i32 @noAlias12( +; CHECK: store <4 x i32> ; CHECK: ret define i32 @noAlias12(i32 %a) #0 { @@ -672,7 +671,7 @@ for.end: ; preds = %for.cond ; Foo.A[i] = Foo.A[i+4] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias13 +; CHECK-LABEL: define i32 @noAlias13( ; CHECK: add nsw <4 x i32> ; CHECK: ret @@ -721,7 +720,7 @@ for.end: ; preds = %for.cond ; Foo.A[SIZE-i-1] = Foo.A[SIZE-i-5] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @noAlias14 +; CHECK-LABEL: define i32 @noAlias14( ; CHECK: sub nsw <4 x i32> ; CHECK: ret @@ -777,7 +776,7 @@ for.end: ; preds = %for.cond ; Foo.A[i] = Foo.B[SIZE-i-1] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @mayAlias01 +; CHECK-LABEL: define i32 @mayAlias01( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret @@ -827,7 +826,7 @@ for.end: ; preds = %for.cond ; Foo.A[SIZE-i-1] = Foo.B[i] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @mayAlias02 +; CHECK-LABEL: define i32 @mayAlias02( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret @@ -877,7 +876,7 @@ for.end: ; preds = %for.cond ; *(PA+i) = *(PB+SIZE-i-1) + a; ; return *(PA+a); ; } -; CHECK: define i32 @mayAlias03 +; CHECK-LABEL: define i32 @mayAlias03( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret @@ -934,7 +933,7 @@ for.end: ; preds = %for.cond ; Foo.A[i+10] = Foo.B[SIZE-i-1] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @mustAlias01 +; CHECK-LABEL: define i32 @mustAlias01( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret @@ -984,7 +983,7 @@ for.end: ; preds = %for.cond ; Foo.A[i] = Foo.B[SIZE-i-10] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @mustAlias02 +; CHECK-LABEL: define i32 @mustAlias02( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret @@ -1033,7 +1032,7 @@ for.end: ; preds = %for.cond ; Foo.A[i+10] = Foo.B[SIZE-i-10] + a; ; return Foo.A[a]; ; } -; CHECK: define i32 @mustAlias03 +; CHECK-LABEL: define i32 @mustAlias03( ; CHECK-NOT: add nsw <4 x i32> ; CHECK: ret diff --git a/test/Transforms/LoopVectorize/hoist-loads.ll b/test/Transforms/LoopVectorize/hoist-loads.ll new file mode 100644 index 0000000000000..765e14d6985ba --- /dev/null +++ b/test/Transforms/LoopVectorize/hoist-loads.ll @@ -0,0 +1,69 @@ +; RUN: opt -loop-vectorize -force-vector-width=2 -force-vector-unroll=1 -S < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +@A = common global [1024 x float] zeroinitializer, align 16 +@B = common global [1024 x float] zeroinitializer, align 16 + +; Make sure we can vectorize in the presence of hoistable conditional loads. +; CHECK-LABEL: @hoist_cond_load( +; CHECK: load <2 x float> + +define void @hoist_cond_load() { +entry: + br label %for.body +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ] + %arrayidx = getelementptr inbounds [1024 x float]* @A, i64 0, i64 %indvars.iv + %arrayidx2 = getelementptr inbounds [1024 x float]* @B, i64 0, i64 %indvars.iv + %0 = load float* %arrayidx2, align 4 + %cmp3 = fcmp oeq float %0, 0.000000e+00 + br i1 %cmp3, label %if.end9, label %if.else + +if.else: + %1 = load float* %arrayidx, align 4 + br label %if.end9 + +if.end9: + %tmp.0 = phi float [ %1, %if.else ], [ 0.000000e+00, %for.body ] + store float %tmp.0, float* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; However, we can't hoist loads whose address we have not seen unconditionally +; accessed. +; CHECK-LABEL: @dont_hoist_cond_load( +; CHECK-NOT: load <2 x float> + +define void @dont_hoist_cond_load() { +entry: + br label %for.body +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %if.end9 ] + %arrayidx = getelementptr inbounds [1024 x float]* @A, i64 0, i64 %indvars.iv + %arrayidx2 = getelementptr inbounds [1024 x float]* @B, i64 0, i64 %indvars.iv + %0 = load float* %arrayidx2, align 4 + %cmp3 = fcmp oeq float %0, 0.000000e+00 + br i1 %cmp3, label %if.end9, label %if.else + +if.else: + %1 = load float* %arrayidx, align 4 + br label %if.end9 + +if.end9: + %tmp.0 = phi float [ %1, %if.else ], [ 0.000000e+00, %for.body ] + store float %tmp.0, float* %arrayidx2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/if-conv-crash.ll b/test/Transforms/LoopVectorize/if-conv-crash.ll index 3283456aa3c3a..f8f2cf1fff4bb 100644 --- a/test/Transforms/LoopVectorize/if-conv-crash.ll +++ b/test/Transforms/LoopVectorize/if-conv-crash.ll @@ -37,3 +37,25 @@ if.end21: ; preds = %lor.lhs.false if.end25: ; preds = %entry ret void } + +; PR15990 +; We can have basic blocks with single entry PHI nodes. +define void @single_entry_phi(i32* %a, i32 *%b) { +entry: + br label %for.cond1.preheader + +for.cond1.preheader: + %inc10 = phi i32 [ 0, %entry ], [ %inc, %for.end ] + br label %for.end + +for.end: + %malicious.phi = phi i32 [ 0, %for.cond1.preheader ] + %inc = add nsw i32 %inc10, 1 + %tobool = icmp eq i32 %inc, 0 + br i1 %tobool, label %for.cond.for.end5, label %for.cond1.preheader + +for.cond.for.end5: + %and.lcssa = phi i32 [ %malicious.phi, %for.end ] + store i32 %and.lcssa, i32* %a, align 4 + ret void +} diff --git a/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll b/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll new file mode 100644 index 0000000000000..27c274d557eb1 --- /dev/null +++ b/test/Transforms/LoopVectorize/if-conversion-edgemasks.ll @@ -0,0 +1,243 @@ +; RUN: opt -S -loop-vectorize < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.9.0" + +@a = global i32* null, align 8 +@b = global i32* null, align 8 +@c = global i32* null, align 8 + +; Don't create an exponetial IR for the edge masks needed when if-converting +; this code. + +; PR16472 + +; CHECK-NOT: %6000000 = + +define void @_Z3fn4i(i32 %p1) { +entry: + %cmp88 = icmp sgt i32 %p1, 0 + br i1 %cmp88, label %for.body.lr.ph, label %for.end + +for.body.lr.ph: + %0 = load i32** @b, align 8 %1 = load i32** @a, align 8 %2 = load i32** @c, align 8 br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %_ZL3fn3ii.exit58 ] + %arrayidx = getelementptr inbounds i32* %0, i64 %indvars.iv + %3 = load i32* %arrayidx, align 4 %4 = trunc i64 %indvars.iv to i32 + %and.i = and i32 %4, 1 + %tobool.i.i = icmp eq i32 %and.i, 0 + br i1 %tobool.i.i, label %if.end.i, label %if.then.i + +if.then.i: + %and.i.i = lshr i32 %3, 2 + %and.lobit.i.i = and i32 %and.i.i, 1 + %5 = xor i32 %and.lobit.i.i, 1 + %or.i.i = or i32 %5, %3 + %cmp.i = icmp sgt i32 %or.i.i, 0 + %conv.i = zext i1 %cmp.i to i32 + br label %if.end.i + +if.end.i: + %tobool.i87 = phi i1 [ true, %if.then.i ], [ false, %for.body ] + %p1.addr.0.i = phi i32 [ %conv.i, %if.then.i ], [ %3, %for.body ] + %6 = trunc i64 %indvars.iv to i32 + %and1.i = and i32 %6, 7 + %tobool2.i = icmp eq i32 %and1.i, 0 + br i1 %tobool2.i, label %if.end7.i, label %if.then3.i + +if.then3.i: + %p1.addr.0.lobit.i = lshr i32 %p1.addr.0.i, 31 + %and6.i = and i32 %p1.addr.0.i, 1 + %or.i = or i32 %p1.addr.0.lobit.i, %and6.i + br label %if.end7.i + +if.end7.i: + %p1.addr.1.i = phi i32 [ %or.i, %if.then3.i ], [ %p1.addr.0.i, %if.end.i ] + br i1 %tobool.i87, label %if.then10.i, label %if.end13.i + +if.then10.i: + %cmp11.i = icmp sgt i32 %p1.addr.1.i, 0 + %conv12.i = zext i1 %cmp11.i to i32 + br label %if.end13.i + +if.end13.i: + %p1.addr.2.i = phi i32 [ %conv12.i, %if.then10.i ], [ %p1.addr.1.i, %if.end7.i ] + br i1 %tobool.i.i, label %_Z3fn2iii.exit, label %if.then16.i + +if.then16.i: + %and17.i = lshr i32 %p1.addr.2.i, 3 + %and17.lobit.i = and i32 %and17.i, 1 + br label %_Z3fn2iii.exit + +_Z3fn2iii.exit: + %p1.addr.3.i = phi i32 [ %and17.lobit.i, %if.then16.i ], [ %p1.addr.2.i, %if.end13.i ] + %7 = trunc i64 %indvars.iv to i32 + %shr.i = ashr i32 %7, 1 + %and.i18.i = and i32 %shr.i, 1 + %tobool.i19.i = icmp ne i32 %and.i18.i, 0 + br i1 %tobool.i19.i, label %if.then.i20.i, label %if.end.i.i + +if.then.i20.i: + %cmp.i.i = icmp sgt i32 %p1.addr.3.i, 0 + %conv.i.i = zext i1 %cmp.i.i to i32 + br label %if.end.i.i + +if.end.i.i: + %p1.addr.0.i21.i = phi i32 [ %conv.i.i, %if.then.i20.i ], [ %p1.addr.3.i, %_Z3fn2iii.exit ] + %and1.i.i = and i32 %shr.i, 7 + %tobool2.i.i = icmp eq i32 %and1.i.i, 0 + br i1 %tobool2.i.i, label %if.end7.i.i, label %if.then3.i.i + +if.then3.i.i: + %p1.addr.0.lobit.i.i = lshr i32 %p1.addr.0.i21.i, 31 + %and6.i.i = and i32 %p1.addr.0.i21.i, 1 + %or.i22.i = or i32 %p1.addr.0.lobit.i.i, %and6.i.i + br label %if.end7.i.i + +if.end7.i.i: + %p1.addr.1.i.i = phi i32 [ %or.i22.i, %if.then3.i.i ], [ %p1.addr.0.i21.i, %if.end.i.i ] + br i1 %tobool.i19.i, label %if.then10.i.i, label %if.end13.i.i + +if.then10.i.i: + %cmp11.i.i = icmp sgt i32 %p1.addr.1.i.i, 0 + %conv12.i.i = zext i1 %cmp11.i.i to i32 + br label %if.end13.i.i + +if.end13.i.i: + %p1.addr.2.i.i = phi i32 [ %conv12.i.i, %if.then10.i.i ], [ %p1.addr.1.i.i, %if.end7.i.i ] + %and14.i.i = and i32 %shr.i, 5 + %tobool15.i.i = icmp eq i32 %and14.i.i, 0 + br i1 %tobool15.i.i, label %_Z3fn2iii.exit.i, label %if.then16.i.i + +if.then16.i.i: + %and17.i.i = lshr i32 %p1.addr.2.i.i, 3 + %and17.lobit.i.i = and i32 %and17.i.i, 1 + br label %_Z3fn2iii.exit.i + +_Z3fn2iii.exit.i: + %p1.addr.3.i.i = phi i32 [ %and17.lobit.i.i, %if.then16.i.i ], [ %p1.addr.2.i.i, %if.end13.i.i ] + %8 = trunc i64 %indvars.iv to i32 + %tobool.i11.i = icmp eq i32 %8, 0 + br i1 %tobool.i11.i, label %_ZL3fn3ii.exit, label %if.then.i15.i + +if.then.i15.i: + %and.i12.i = lshr i32 %p1.addr.3.i.i, 2 + %and.lobit.i13.i = and i32 %and.i12.i, 1 + %9 = xor i32 %and.lobit.i13.i, 1 + %or.i14.i = or i32 %9, %p1.addr.3.i.i + br label %_ZL3fn3ii.exit + +_ZL3fn3ii.exit: + %p1.addr.0.i16.i = phi i32 [ %or.i14.i, %if.then.i15.i ], [ %p1.addr.3.i.i, %_Z3fn2iii.exit.i ] + %arrayidx2 = getelementptr inbounds i32* %1, i64 %indvars.iv + store i32 %p1.addr.0.i16.i, i32* %arrayidx2, align 4 %arrayidx4 = getelementptr inbounds i32* %0, i64 %indvars.iv + %10 = load i32* %arrayidx4, align 4 br i1 %tobool.i.i, label %_Z3fn1ii.exit.i26, label %if.then.i.i21 + +if.then.i.i21: + %and.i.i18 = lshr i32 %10, 2 + %and.lobit.i.i19 = and i32 %and.i.i18, 1 + %11 = xor i32 %and.lobit.i.i19, 1 + %or.i.i20 = or i32 %11, %10 + br label %_Z3fn1ii.exit.i26 + +_Z3fn1ii.exit.i26: + %p1.addr.0.i.i22 = phi i32 [ %or.i.i20, %if.then.i.i21 ], [ %10, %_ZL3fn3ii.exit ] + br i1 %tobool.i87, label %if.then.i63, label %if.end.i67 + +if.then.i63: + %cmp.i61 = icmp sgt i32 %p1.addr.0.i.i22, 0 + %conv.i62 = zext i1 %cmp.i61 to i32 + br label %if.end.i67 + +if.end.i67: + %p1.addr.0.i64 = phi i32 [ %conv.i62, %if.then.i63 ], [ %p1.addr.0.i.i22, %_Z3fn1ii.exit.i26 ] + br i1 %tobool2.i, label %if.end7.i73, label %if.then3.i71 + +if.then3.i71: + %p1.addr.0.lobit.i68 = lshr i32 %p1.addr.0.i64, 31 + %and6.i69 = and i32 %p1.addr.0.i64, 1 + %or.i70 = or i32 %p1.addr.0.lobit.i68, %and6.i69 + br label %if.end7.i73 + +if.end7.i73: + %p1.addr.1.i72 = phi i32 [ %or.i70, %if.then3.i71 ], [ %p1.addr.0.i64, %if.end.i67 ] + br i1 %tobool.i87, label %if.then10.i76, label %if.end13.i80 + +if.then10.i76: + %cmp11.i74 = icmp sgt i32 %p1.addr.1.i72, 0 + %conv12.i75 = zext i1 %cmp11.i74 to i32 + br label %if.end13.i80 + +if.end13.i80: + %p1.addr.2.i77 = phi i32 [ %conv12.i75, %if.then10.i76 ], [ %p1.addr.1.i72, %if.end7.i73 ] + br i1 %tobool.i.i, label %_Z3fn2iii.exit85, label %if.then16.i83 + +if.then16.i83: + %and17.i81 = lshr i32 %p1.addr.2.i77, 3 + %and17.lobit.i82 = and i32 %and17.i81, 1 + br label %_Z3fn2iii.exit85 + +_Z3fn2iii.exit85: + %p1.addr.3.i84 = phi i32 [ %and17.lobit.i82, %if.then16.i83 ], [ %p1.addr.2.i77, %if.end13.i80 ] + br i1 %tobool.i19.i, label %if.then.i20.i29, label %if.end.i.i33 + +if.then.i20.i29: + %cmp.i.i27 = icmp sgt i32 %p1.addr.3.i84, 0 + %conv.i.i28 = zext i1 %cmp.i.i27 to i32 + br label %if.end.i.i33 + +if.end.i.i33: + %p1.addr.0.i21.i30 = phi i32 [ %conv.i.i28, %if.then.i20.i29 ], [ %p1.addr.3.i84, %_Z3fn2iii.exit85 ] + br i1 %tobool2.i.i, label %if.end7.i.i39, label %if.then3.i.i37 + +if.then3.i.i37: + %p1.addr.0.lobit.i.i34 = lshr i32 %p1.addr.0.i21.i30, 31 + %and6.i.i35 = and i32 %p1.addr.0.i21.i30, 1 + %or.i22.i36 = or i32 %p1.addr.0.lobit.i.i34, %and6.i.i35 + br label %if.end7.i.i39 + +if.end7.i.i39: + %p1.addr.1.i.i38 = phi i32 [ %or.i22.i36, %if.then3.i.i37 ], [ %p1.addr.0.i21.i30, %if.end.i.i33 ] + br i1 %tobool.i19.i, label %if.then10.i.i42, label %if.end13.i.i46 + +if.then10.i.i42: + %cmp11.i.i40 = icmp sgt i32 %p1.addr.1.i.i38, 0 + %conv12.i.i41 = zext i1 %cmp11.i.i40 to i32 + br label %if.end13.i.i46 + +if.end13.i.i46: + %p1.addr.2.i.i43 = phi i32 [ %conv12.i.i41, %if.then10.i.i42 ], [ %p1.addr.1.i.i38, %if.end7.i.i39 ] + br i1 %tobool15.i.i, label %_Z3fn2iii.exit.i52, label %if.then16.i.i49 + +if.then16.i.i49: + %and17.i.i47 = lshr i32 %p1.addr.2.i.i43, 3 + %and17.lobit.i.i48 = and i32 %and17.i.i47, 1 + br label %_Z3fn2iii.exit.i52 + +_Z3fn2iii.exit.i52: + %p1.addr.3.i.i50 = phi i32 [ %and17.lobit.i.i48, %if.then16.i.i49 ], [ %p1.addr.2.i.i43, %if.end13.i.i46 ] + br i1 %tobool.i11.i, label %_ZL3fn3ii.exit58, label %if.then.i15.i56 + +if.then.i15.i56: + %and.i12.i53 = lshr i32 %p1.addr.3.i.i50, 2 + %and.lobit.i13.i54 = and i32 %and.i12.i53, 1 + %12 = xor i32 %and.lobit.i13.i54, 1 + %or.i14.i55 = or i32 %12, %p1.addr.3.i.i50 + br label %_ZL3fn3ii.exit58 + +_ZL3fn3ii.exit58: + %p1.addr.0.i16.i57 = phi i32 [ %or.i14.i55, %if.then.i15.i56 ], [ %p1.addr.3.i.i50, %_Z3fn2iii.exit.i52 ] + %arrayidx7 = getelementptr inbounds i32* %2, i64 %indvars.iv + store i32 %p1.addr.0.i16.i57, i32* %arrayidx7, align 4 %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, %p1 + br i1 %exitcond, label %for.body, label %for.cond.for.end_crit_edge + +for.cond.for.end_crit_edge: + br label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/if-conversion-nest.ll b/test/Transforms/LoopVectorize/if-conversion-nest.ll index f44862a2ebb92..92cb06e5e5251 100644 --- a/test/Transforms/LoopVectorize/if-conversion-nest.ll +++ b/test/Transforms/LoopVectorize/if-conversion-nest.ll @@ -2,7 +2,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK: icmp sgt ;CHECK: icmp sgt ;CHECK: icmp slt diff --git a/test/Transforms/LoopVectorize/if-conversion-reduction.ll b/test/Transforms/LoopVectorize/if-conversion-reduction.ll index 3a2d82e15d63a..8cb703cdfa4b6 100644 --- a/test/Transforms/LoopVectorize/if-conversion-reduction.ll +++ b/test/Transforms/LoopVectorize/if-conversion-reduction.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.9.0" -;CHECK: @reduction_func +;CHECK-LABEL: @reduction_func( ;CHECK-NOT: load <4 x i32> ;CHECK: ret i32 define i32 @reduction_func(i32* nocapture %A, i32 %n) nounwind uwtable readonly ssp { diff --git a/test/Transforms/LoopVectorize/if-conversion.ll b/test/Transforms/LoopVectorize/if-conversion.ll index 6e7c03a556c4e..dbe0243a81106 100644 --- a/test/Transforms/LoopVectorize/if-conversion.ll +++ b/test/Transforms/LoopVectorize/if-conversion.ll @@ -17,7 +17,7 @@ target triple = "x86_64-apple-macosx10.9.0" ; } ;} -;CHECK: @function0 +;CHECK-LABEL: @function0( ;CHECK: load <4 x i32> ;CHECK: icmp sgt <4 x i32> ;CHECK: mul <4 x i32> @@ -70,7 +70,7 @@ for.end: ; return sum; ; } -;CHECK: @reduction_func +;CHECK-LABEL: @reduction_func( ;CHECK: load <4 x i32> ;CHECK: icmp sgt <4 x i32> ;CHECK: add <4 x i32> @@ -106,3 +106,66 @@ for.end: ; preds = %for.inc, %entry ret i32 %sum.0.lcssa } +@a = common global [1 x i32*] zeroinitializer, align 8 +@c = common global i32* null, align 8 + +; We use to if convert this loop. This is not safe because there is a trapping +; constant expression. +; PR16729 + +; CHECK-LABEL: trapping_constant_expression +; CHECK-NOT: or <4 x i32> + +define i32 @trapping_constant_expression() { +entry: + br label %for.body + +for.body: + %inc3 = phi i32 [ 0, %entry ], [ %inc, %cond.end ] + %or2 = phi i32 [ 0, %entry ], [ %or, %cond.end ] + br i1 icmp eq (i32** getelementptr inbounds ([1 x i32*]* @a, i64 0, i64 0), i32** @c), label %cond.false, label %cond.end + +cond.false: + br label %cond.end + +cond.end: + %cond = phi i32 [ sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*]* @a, i64 0, i64 0), i32** @c) to i32)), %cond.false ], [ 0, %for.body ] + %or = or i32 %or2, %cond + %inc = add nsw i32 %inc3, 1 + %cmp = icmp slt i32 %inc, 128 + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret i32 %or +} + +; Neither should we if-convert if there is an instruction operand that is a +; trapping constant expression. +; PR16729 + +; CHECK-LABEL: trapping_constant_expression2 +; CHECK-NOT: or <4 x i32> + +define i32 @trapping_constant_expression2() { +entry: + br label %for.body + +for.body: + %inc3 = phi i32 [ 0, %entry ], [ %inc, %cond.end ] + %or2 = phi i32 [ 0, %entry ], [ %or, %cond.end ] + br i1 icmp eq (i32** getelementptr inbounds ([1 x i32*]* @a, i64 0, i64 0), i32** @c), label %cond.false, label %cond.end + +cond.false: + %cond.1 = or i32 %inc3, sdiv (i32 1, i32 zext (i1 icmp eq (i32** getelementptr inbounds ([1 x i32*]* @a, i64 0, i64 0), i32** @c) to i32)) + br label %cond.end + +cond.end: + %cond = phi i32 [ %cond.1, %cond.false ], [ %inc3, %for.body ] + %or = or i32 %or2, %cond + %inc = add nsw i32 %inc3, 1 + %cmp = icmp slt i32 %inc, 128 + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret i32 %or +} diff --git a/test/Transforms/LoopVectorize/increment.ll b/test/Transforms/LoopVectorize/increment.ll index 3fa6b19ca9280..d35bd58a0281f 100644 --- a/test/Transforms/LoopVectorize/increment.ll +++ b/test/Transforms/LoopVectorize/increment.ll @@ -9,7 +9,7 @@ target triple = "x86_64-apple-macosx10.8.0" ; for (i=0; i<n; i++){ ; a[i] += i; ; } -;CHECK: @inc +;CHECK-LABEL: @inc( ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> ;CHECK: store <4 x i32> @@ -39,7 +39,7 @@ define void @inc(i32 %n) nounwind uwtable noinline ssp { ; for (i = 0; i < n; ++i) { ; A[B[i]]++; ; -;CHECK: @histogram +;CHECK-LABEL: @histogram( ;CHECK-NOT: <4 x i32> ;CHECK: ret i32 define i32 @histogram(i32* nocapture noalias %A, i32* nocapture noalias %B, i32 %n) nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/induction.ll b/test/Transforms/LoopVectorize/induction.ll new file mode 100644 index 0000000000000..50c3b6b6e79b2 --- /dev/null +++ b/test/Transforms/LoopVectorize/induction.ll @@ -0,0 +1,110 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=2 -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; Make sure that we can handle multiple integer induction variables. +; CHECK-LABEL: @multi_int_induction( +; CHECK: vector.body: +; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK: %normalized.idx = sub i64 %index, 0 +; CHECK: %[[VAR:.*]] = trunc i64 %normalized.idx to i32 +; CHECK: %offset.idx = add i32 190, %[[VAR]] +define void @multi_int_induction(i32* %A, i32 %N) { +for.body.lr.ph: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next, %for.body ] + %count.09 = phi i32 [ 190, %for.body.lr.ph ], [ %inc, %for.body ] + %arrayidx2 = getelementptr inbounds i32* %A, i64 %indvars.iv + store i32 %count.09, i32* %arrayidx2, align 4 + %inc = add nsw i32 %count.09, 1 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, %N + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=2 -instcombine -S | FileCheck %s --check-prefix=IND + +; Make sure we remove unneeded vectorization of induction variables. +; In order for instcombine to cleanup the vectorized induction variables that we +; create in the loop vectorizer we need to perform some form of redundancy +; elimination to get rid of multiple uses. + +; IND-LABEL: scalar_use + +; IND: br label %vector.body +; IND: vector.body: +; Vectorized induction variable. +; IND-NOT: insertelement <2 x i64> +; IND-NOT: shufflevector <2 x i64> +; IND: br {{.*}}, label %vector.body + +define void @scalar_use(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) { +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %ind.sum = add i64 %iv, %offset + %arr.idx = getelementptr inbounds float* %a, i64 %ind.sum + %l1 = load float* %arr.idx, align 4 + %ind.sum2 = add i64 %iv, %offset2 + %arr.idx2 = getelementptr inbounds float* %a, i64 %ind.sum2 + %l2 = load float* %arr.idx2, align 4 + %m = fmul fast float %b, %l2 + %ad = fadd fast float %l1, %m + store float %ad, float* %arr.idx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %loopexit, label %for.body + +loopexit: + ret void +} + + +; Make sure that the loop exit count computation does not overflow for i8 and +; i16. The exit count of these loops is i8/i16 max + 1. If we don't cast the +; induction variable to a bigger type the exit count computation will overflow +; to 0. +; PR17532 + +; CHECK-LABEL: i8_loop +; CHECK; icmp eq i32 {{.*}}, 256 +define i32 @i8_loop() nounwind readnone ssp uwtable { + br label %1 + +; <label>:1 ; preds = %1, %0 + %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ] + %b.0 = phi i8 [ 0, %0 ], [ %3, %1 ] + %2 = and i32 %a.0, 4 + %3 = add i8 %b.0, -1 + %4 = icmp eq i8 %3, 0 + br i1 %4, label %5, label %1 + +; <label>:5 ; preds = %1 + ret i32 %2 +} + +; CHECK-LABEL: i16_loop +; CHECK; icmp eq i32 {{.*}}, 65536 + +define i32 @i16_loop() nounwind readnone ssp uwtable { + br label %1 + +; <label>:1 ; preds = %1, %0 + %a.0 = phi i32 [ 1, %0 ], [ %2, %1 ] + %b.0 = phi i16 [ 0, %0 ], [ %3, %1 ] + %2 = and i32 %a.0, 4 + %3 = add i16 %b.0, -1 + %4 = icmp eq i16 %3, 0 + br i1 %4, label %5, label %1 + +; <label>:5 ; preds = %1 + ret i32 %2 +} diff --git a/test/Transforms/LoopVectorize/induction_plus.ll b/test/Transforms/LoopVectorize/induction_plus.ll index 96595cdc16bc2..9c8201ab78053 100644 --- a/test/Transforms/LoopVectorize/induction_plus.ll +++ b/test/Transforms/LoopVectorize/induction_plus.ll @@ -5,9 +5,9 @@ target triple = "x86_64-apple-macosx10.8.0" @array = common global [1024 x i32] zeroinitializer, align 16 -;CHECK: @array_at_plus_one -;CHECK: trunc i64 +;CHECK-LABEL: @array_at_plus_one( ;CHECK: add i64 %index, 12 +;CHECK: trunc i64 ;CHECK: ret i32 define i32 @array_at_plus_one(i32 %n) nounwind uwtable ssp { %1 = icmp sgt i32 %n, 0 diff --git a/test/Transforms/LoopVectorize/infiniteloop.ll b/test/Transforms/LoopVectorize/infiniteloop.ll new file mode 100644 index 0000000000000..5c5e1a3be0a24 --- /dev/null +++ b/test/Transforms/LoopVectorize/infiniteloop.ll @@ -0,0 +1,34 @@ +; RUN: opt -S -indvars -loop-vectorize -force-vector-width=2 < %s | FileCheck %s + +target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128" + +@a = common global i64 0, align 8 +@x = common global i32 0, align 4 + +; We used to assert on this loop because we could not find an induction +; variable but assumed there must be one. Scalar evolution returned a exit +; count for the loop below and from there on we assumed that there must be an +; induction variable. This is not a valid assumption: +; // getExitCount - Get the expression for the number of loop iterations for +; // which this loop is *guaranteed not to exit* via ExitingBlock. Otherwise +; // return SCEVCouldNotCompute. +; For an infinite loop SE can return any number. + +; CHECK-LABEL: @fn1( +define void @fn1() { +entry: + store i64 0, i64* @a, align 8 + br label %for.body + +for.body: ; preds = %for.body, %entry + %inc1 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + store volatile i32 0, i32* @x, align 4 + %inc = add nsw i64 %inc1, 1 + %cmp = icmp sgt i64 %inc1, -2 + br i1 %cmp, label %for.body, label %for.end + +for.end: ; preds = %for.body + %inc.lcssa = phi i64 [ %inc, %for.body ] + store i64 %inc.lcssa, i64* @a, align 8 + ret void +} diff --git a/test/Transforms/LoopVectorize/intrinsic.ll b/test/Transforms/LoopVectorize/intrinsic.ll index defbb5bd94ba0..c3d570c03a77f 100644 --- a/test/Transforms/LoopVectorize/intrinsic.ll +++ b/test/Transforms/LoopVectorize/intrinsic.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -;CHECK: @sqrt_f32 +;CHECK-LABEL: @sqrt_f32( ;CHECK: llvm.sqrt.v4f32 ;CHECK: ret void define void @sqrt_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -29,7 +29,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.sqrt.f32(float) nounwind readnone -;CHECK: @sqrt_f64 +;CHECK-LABEL: @sqrt_f64( ;CHECK: llvm.sqrt.v4f64 ;CHECK: ret void define void @sqrt_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -55,7 +55,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.sqrt.f64(double) nounwind readnone -;CHECK: @sin_f32 +;CHECK-LABEL: @sin_f32( ;CHECK: llvm.sin.v4f32 ;CHECK: ret void define void @sin_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -81,7 +81,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.sin.f32(float) nounwind readnone -;CHECK: @sin_f64 +;CHECK-LABEL: @sin_f64( ;CHECK: llvm.sin.v4f64 ;CHECK: ret void define void @sin_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -107,7 +107,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.sin.f64(double) nounwind readnone -;CHECK: @cos_f32 +;CHECK-LABEL: @cos_f32( ;CHECK: llvm.cos.v4f32 ;CHECK: ret void define void @cos_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -133,7 +133,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.cos.f32(float) nounwind readnone -;CHECK: @cos_f64 +;CHECK-LABEL: @cos_f64( ;CHECK: llvm.cos.v4f64 ;CHECK: ret void define void @cos_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -159,7 +159,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.cos.f64(double) nounwind readnone -;CHECK: @exp_f32 +;CHECK-LABEL: @exp_f32( ;CHECK: llvm.exp.v4f32 ;CHECK: ret void define void @exp_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -185,7 +185,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.exp.f32(float) nounwind readnone -;CHECK: @exp_f64 +;CHECK-LABEL: @exp_f64( ;CHECK: llvm.exp.v4f64 ;CHECK: ret void define void @exp_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -211,7 +211,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.exp.f64(double) nounwind readnone -;CHECK: @exp2_f32 +;CHECK-LABEL: @exp2_f32( ;CHECK: llvm.exp2.v4f32 ;CHECK: ret void define void @exp2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -237,7 +237,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.exp2.f32(float) nounwind readnone -;CHECK: @exp2_f64 +;CHECK-LABEL: @exp2_f64( ;CHECK: llvm.exp2.v4f64 ;CHECK: ret void define void @exp2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -263,7 +263,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.exp2.f64(double) nounwind readnone -;CHECK: @log_f32 +;CHECK-LABEL: @log_f32( ;CHECK: llvm.log.v4f32 ;CHECK: ret void define void @log_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -289,7 +289,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.log.f32(float) nounwind readnone -;CHECK: @log_f64 +;CHECK-LABEL: @log_f64( ;CHECK: llvm.log.v4f64 ;CHECK: ret void define void @log_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -315,7 +315,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.log.f64(double) nounwind readnone -;CHECK: @log10_f32 +;CHECK-LABEL: @log10_f32( ;CHECK: llvm.log10.v4f32 ;CHECK: ret void define void @log10_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -341,7 +341,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.log10.f32(float) nounwind readnone -;CHECK: @log10_f64 +;CHECK-LABEL: @log10_f64( ;CHECK: llvm.log10.v4f64 ;CHECK: ret void define void @log10_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -367,7 +367,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.log10.f64(double) nounwind readnone -;CHECK: @log2_f32 +;CHECK-LABEL: @log2_f32( ;CHECK: llvm.log2.v4f32 ;CHECK: ret void define void @log2_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -393,7 +393,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.log2.f32(float) nounwind readnone -;CHECK: @log2_f64 +;CHECK-LABEL: @log2_f64( ;CHECK: llvm.log2.v4f64 ;CHECK: ret void define void @log2_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -419,7 +419,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.log2.f64(double) nounwind readnone -;CHECK: @fabs_f32 +;CHECK-LABEL: @fabs_f32( ;CHECK: llvm.fabs.v4f32 ;CHECK: ret void define void @fabs_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -468,7 +468,60 @@ for.end: ; preds = %for.body, %entry declare double @llvm.fabs(double) nounwind readnone -;CHECK: @floor_f32 +;CHECK-LABEL: @copysign_f32( +;CHECK: llvm.copysign.v4f32 +;CHECK: ret void +define void @copysign_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds float* %z, i64 %indvars.iv + %1 = load float* %arrayidx1, align 4 + %call = tail call float @llvm.copysign.f32(float %0, float %1) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.copysign.f32(float, float) nounwind readnone + +define void @copysign_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8 + %arrayidx1 = getelementptr inbounds double* %z, i64 %indvars.iv + %1 = load double* %arrayidx, align 8 + %call = tail call double @llvm.copysign(double %0, double %1) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.copysign(double, double) nounwind readnone + +;CHECK-LABEL: @floor_f32( ;CHECK: llvm.floor.v4f32 ;CHECK: ret void define void @floor_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -494,7 +547,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.floor.f32(float) nounwind readnone -;CHECK: @floor_f64 +;CHECK-LABEL: @floor_f64( ;CHECK: llvm.floor.v4f64 ;CHECK: ret void define void @floor_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -520,7 +573,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.floor.f64(double) nounwind readnone -;CHECK: @ceil_f32 +;CHECK-LABEL: @ceil_f32( ;CHECK: llvm.ceil.v4f32 ;CHECK: ret void define void @ceil_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -546,7 +599,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.ceil.f32(float) nounwind readnone -;CHECK: @ceil_f64 +;CHECK-LABEL: @ceil_f64( ;CHECK: llvm.ceil.v4f64 ;CHECK: ret void define void @ceil_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -572,7 +625,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.ceil.f64(double) nounwind readnone -;CHECK: @trunc_f32 +;CHECK-LABEL: @trunc_f32( ;CHECK: llvm.trunc.v4f32 ;CHECK: ret void define void @trunc_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -598,7 +651,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.trunc.f32(float) nounwind readnone -;CHECK: @trunc_f64 +;CHECK-LABEL: @trunc_f64( ;CHECK: llvm.trunc.v4f64 ;CHECK: ret void define void @trunc_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -624,7 +677,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.trunc.f64(double) nounwind readnone -;CHECK: @rint_f32 +;CHECK-LABEL: @rint_f32( ;CHECK: llvm.rint.v4f32 ;CHECK: ret void define void @rint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -650,7 +703,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.rint.f32(float) nounwind readnone -;CHECK: @rint_f64 +;CHECK-LABEL: @rint_f64( ;CHECK: llvm.rint.v4f64 ;CHECK: ret void define void @rint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -676,7 +729,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.rint.f64(double) nounwind readnone -;CHECK: @nearbyint_f32 +;CHECK-LABEL: @nearbyint_f32( ;CHECK: llvm.nearbyint.v4f32 ;CHECK: ret void define void @nearbyint_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { @@ -702,7 +755,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.nearbyint.f32(float) nounwind readnone -;CHECK: @nearbyint_f64 +;CHECK-LABEL: @nearbyint_f64( ;CHECK: llvm.nearbyint.v4f64 ;CHECK: ret void define void @nearbyint_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { @@ -728,7 +781,59 @@ for.end: ; preds = %for.body, %entry declare double @llvm.nearbyint.f64(double) nounwind readnone -;CHECK: @fma_f32 +;CHECK-LABEL: @round_f32( +;CHECK: llvm.round.v4f32 +;CHECK: ret void +define void @round_f32(i32 %n, float* noalias %y, float* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds float* %y, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %call = tail call float @llvm.round.f32(float %0) nounwind readnone + %arrayidx2 = getelementptr inbounds float* %x, i64 %indvars.iv + store float %call, float* %arrayidx2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare float @llvm.round.f32(float) nounwind readnone + +;CHECK-LABEL: @round_f64( +;CHECK: llvm.round.v4f64 +;CHECK: ret void +define void @round_f64(i32 %n, double* noalias %y, double* noalias %x) nounwind uwtable { +entry: + %cmp6 = icmp sgt i32 %n, 0 + br i1 %cmp6, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds double* %y, i64 %indvars.iv + %0 = load double* %arrayidx, align 8 + %call = tail call double @llvm.round.f64(double %0) nounwind readnone + %arrayidx2 = getelementptr inbounds double* %x, i64 %indvars.iv + store double %call, double* %arrayidx2, align 8 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +declare double @llvm.round.f64(double) nounwind readnone + +;CHECK-LABEL: @fma_f32( ;CHECK: llvm.fma.v4f32 ;CHECK: ret void define void @fma_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z, float* noalias %w) nounwind uwtable { @@ -758,7 +863,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.fma.f32(float, float, float) nounwind readnone -;CHECK: @fma_f64 +;CHECK-LABEL: @fma_f64( ;CHECK: llvm.fma.v4f64 ;CHECK: ret void define void @fma_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z, double* noalias %w) nounwind uwtable { @@ -788,7 +893,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.fma.f64(double, double, double) nounwind readnone -;CHECK: @fmuladd_f32 +;CHECK-LABEL: @fmuladd_f32( ;CHECK: llvm.fmuladd.v4f32 ;CHECK: ret void define void @fmuladd_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z, float* noalias %w) nounwind uwtable { @@ -818,7 +923,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.fmuladd.f32(float, float, float) nounwind readnone -;CHECK: @fmuladd_f64 +;CHECK-LABEL: @fmuladd_f64( ;CHECK: llvm.fmuladd.v4f64 ;CHECK: ret void define void @fmuladd_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z, double* noalias %w) nounwind uwtable { @@ -848,7 +953,7 @@ for.end: ; preds = %for.body, %entry declare double @llvm.fmuladd.f64(double, double, double) nounwind readnone -;CHECK: @pow_f32 +;CHECK-LABEL: @pow_f32( ;CHECK: llvm.pow.v4f32 ;CHECK: ret void define void @pow_f32(i32 %n, float* noalias %y, float* noalias %x, float* noalias %z) nounwind uwtable { @@ -876,7 +981,7 @@ for.end: ; preds = %for.body, %entry declare float @llvm.pow.f32(float, float) nounwind readnone -;CHECK: @pow_f64 +;CHECK-LABEL: @pow_f64( ;CHECK: llvm.pow.v4f64 ;CHECK: ret void define void @pow_f64(i32 %n, double* noalias %y, double* noalias %x, double* noalias %z) nounwind uwtable { @@ -927,3 +1032,61 @@ for.end: ; preds = %for.body declare float @fabsf(float) nounwind readnone declare double @llvm.pow.f64(double, double) nounwind readnone + + + +; Make sure we don't replace calls to functions with standard library function +; signatures but defined with internal linkage. + +define internal float @roundf(float %x) nounwind readnone { + ret float 0.00000000 +} +; CHECK-LABEL: internal_round +; CHECK-NOT: load <4 x float> + +define void @internal_round(float* nocapture %x) nounwind { +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds float* %x, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %call = tail call float @roundf(float %0) nounwind readnone + store float %call, float* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} + +; Make sure we don't replace calls to functions with standard library names but +; different signatures. + +declare void @round(double %f) + +; CHECK-LABEL: wrong_signature +; CHECK-NOT: load <4 x double> + +define void @wrong_signature(double* nocapture %x) nounwind { +entry: + br label %for.body + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds double* %x, i64 %indvars.iv + %0 = load double* %arrayidx, align 4 + store double %0, double* %arrayidx, align 4 + tail call void @round(double %0) nounwind readnone + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} + diff --git a/test/Transforms/LoopVectorize/lifetime.ll b/test/Transforms/LoopVectorize/lifetime.ll new file mode 100644 index 0000000000000..4f6f3b820a437 --- /dev/null +++ b/test/Transforms/LoopVectorize/lifetime.ll @@ -0,0 +1,96 @@ +; RUN: opt -S -loop-vectorize -force-vector-width=2 -force-vector-unroll=1 < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; Make sure we can vectorize loops which contain lifetime markers. + +; CHECK-LABEL: @test( +; CHECK: call void @llvm.lifetime.end +; CHECK: store <2 x i32> +; CHECK: call void @llvm.lifetime.start + +define void @test(i32 *%d) { +entry: + %arr = alloca [1024 x i32], align 16 + %0 = bitcast [1024 x i32]* %arr to i8* + call void @llvm.lifetime.start(i64 4096, i8* %0) #1 + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + call void @llvm.lifetime.end(i64 4096, i8* %0) #1 + %arrayidx = getelementptr inbounds i32* %d, i64 %indvars.iv + %1 = load i32* %arrayidx, align 8 + store i32 100, i32* %arrayidx, align 8 + call void @llvm.lifetime.start(i64 4096, i8* %0) #1 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + call void @llvm.lifetime.end(i64 4096, i8* %0) #1 + ret void +} + +; CHECK-LABEL: @testbitcast( +; CHECK: call void @llvm.lifetime.end +; CHECK: store <2 x i32> +; CHECK: call void @llvm.lifetime.start + +define void @testbitcast(i32 *%d) { +entry: + %arr = alloca [1024 x i32], align 16 + %0 = bitcast [1024 x i32]* %arr to i8* + call void @llvm.lifetime.start(i64 4096, i8* %0) #1 + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %1 = bitcast [1024 x i32]* %arr to i8* + call void @llvm.lifetime.end(i64 4096, i8* %1) #1 + %arrayidx = getelementptr inbounds i32* %d, i64 %indvars.iv + %2 = load i32* %arrayidx, align 8 + store i32 100, i32* %arrayidx, align 8 + call void @llvm.lifetime.start(i64 4096, i8* %1) #1 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + call void @llvm.lifetime.end(i64 4096, i8* %0) #1 + ret void +} + +; CHECK-LABEL: @testloopvariant( +; CHECK: call void @llvm.lifetime.end +; CHECK: store <2 x i32> +; CHECK: call void @llvm.lifetime.start + +define void @testloopvariant(i32 *%d) { +entry: + %arr = alloca [1024 x i32], align 16 + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %0 = getelementptr [1024 x i32]* %arr, i32 0, i64 %indvars.iv + %1 = bitcast [1024 x i32]* %arr to i8* + call void @llvm.lifetime.end(i64 4096, i8* %1) #1 + %arrayidx = getelementptr inbounds i32* %d, i64 %indvars.iv + %2 = load i32* %arrayidx, align 8 + store i32 100, i32* %arrayidx, align 8 + call void @llvm.lifetime.start(i64 4096, i8* %1) #1 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +declare void @llvm.lifetime.start(i64, i8* nocapture) #1 + +declare void @llvm.lifetime.end(i64, i8* nocapture) #1 diff --git a/test/Transforms/LoopVectorize/lit.local.cfg b/test/Transforms/LoopVectorize/lit.local.cfg deleted file mode 100644 index 19eebc0ac7ac3..0000000000000 --- a/test/Transforms/LoopVectorize/lit.local.cfg +++ /dev/null @@ -1 +0,0 @@ -config.suffixes = ['.ll', '.c', '.cpp'] diff --git a/test/Transforms/LoopVectorize/memdep.ll b/test/Transforms/LoopVectorize/memdep.ll new file mode 100644 index 0000000000000..21cb703ba47c0 --- /dev/null +++ b/test/Transforms/LoopVectorize/memdep.ll @@ -0,0 +1,222 @@ +; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-unroll=1 -S | FileCheck %s +; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-unroll=1 -S | FileCheck %s -check-prefix=WIDTH + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; Vectorization with dependence checks. + +; No plausible dependence - can be vectorized. +; for (i = 0; i < 1024; ++i) +; A[i] = A[i + 1] + 1; + +; CHECK-LABEL: @f1_vec( +; CHECK: <2 x i32> + +define void @f1_vec(i32* %A) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %indvars.iv.next = add i32 %indvars.iv, 1 + %arrayidx = getelementptr inbounds i32* %A, i32 %indvars.iv.next + %0 = load i32* %arrayidx, align 4 + %add1 = add nsw i32 %0, 1 + %arrayidx3 = getelementptr inbounds i32* %A, i32 %indvars.iv + store i32 %add1, i32* %arrayidx3, align 4 + %exitcond = icmp ne i32 %indvars.iv.next, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; Plausible dependence of distance 1 - can't be vectorized. +; for (i = 0; i < 1024; ++i) +; A[i+1] = A[i] + 1; + +; CHECK-LABEL: @f2_novec( +; CHECK-NOT: <2 x i32> + +define void @f2_novec(i32* %A) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i32 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %A, i32 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %add = add nsw i32 %0, 1 + %indvars.iv.next = add i32 %indvars.iv, 1 + %arrayidx3 = getelementptr inbounds i32* %A, i32 %indvars.iv.next + store i32 %add, i32* %arrayidx3, align 4 + %exitcond = icmp ne i32 %indvars.iv.next, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; Plausible dependence of distance 2 - can be vectorized with a width of 2. +; for (i = 0; i < 1024; ++i) +; A[i+2] = A[i] + 1; + +; CHECK-LABEL: @f3_vec_len( +; CHECK: <2 x i32> + +; WIDTH: f3_vec_len +; WIDTH-NOT: <4 x i32> + +define void @f3_vec_len(i32* %A) { +entry: + br label %for.body + +for.body: + %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %idxprom = sext i32 %i.01 to i64 + %arrayidx = getelementptr inbounds i32* %A, i64 %idxprom + %0 = load i32* %arrayidx, align 4 + %add = add nsw i32 %0, 1 + %add1 = add nsw i32 %i.01, 2 + %idxprom2 = sext i32 %add1 to i64 + %arrayidx3 = getelementptr inbounds i32* %A, i64 %idxprom2 + store i32 %add, i32* %arrayidx3, align 4 + %inc = add nsw i32 %i.01, 1 + %cmp = icmp slt i32 %inc, 1024 + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret void +} + +; Plausible dependence of distance 1 - cannot be vectorized (without reordering +; accesses). +; for (i = 0; i < 1024; ++i) { +; B[i] = A[i]; +; A[i] = B[i + 1]; +; } + +; CHECK-LABEL: @f5( +; CHECK-NOT: <2 x i32> + +define void @f5(i32* %A, i32* %B) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %A, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds i32* %B, i64 %indvars.iv + store i32 %0, i32* %arrayidx2, align 4 + %indvars.iv.next = add nsw i64 %indvars.iv, 1 + %arrayidx4 = getelementptr inbounds i32* %B, i64 %indvars.iv.next + %1 = load i32* %arrayidx4, align 4 + store i32 %1, i32* %arrayidx, align 4 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; Dependence through a phi node - must not vectorize. +; for (i = 0; i < 1024; ++i) { +; a[i+1] = tmp; +; tmp = a[i]; +; } + +; CHECK-LABEL: @f6 +; CHECK-NOT: <2 x i32> + +define i32 @f6(i32* %a, i32 %tmp) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %tmp.addr.08 = phi i32 [ %tmp, %entry ], [ %0, %for.body ] + %indvars.iv.next = add nsw i64 %indvars.iv, 1 + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv.next + store i32 %tmp.addr.08, i32* %arrayidx, align 4 + %arrayidx3 = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx3, align 4 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 1024 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret i32 undef +} + +; Don't vectorize true loop carried dependencies that are not a multiple of the +; vector width. +; Example: +; for (int i = ...; ++i) { +; a[i] = a[i-3] + ...; +; It is a bad idea to vectorize this loop because store-load forwarding will not +; happen. +; + +; CHECK-LABEL: @nostoreloadforward( +; CHECK-NOT: <2 x i32> + +define void @nostoreloadforward(i32* %A) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ] + %0 = add nsw i64 %indvars.iv, -3 + %arrayidx = getelementptr inbounds i32* %A, i64 %0 + %1 = load i32* %arrayidx, align 4 + %2 = add nsw i64 %indvars.iv, 4 + %arrayidx2 = getelementptr inbounds i32* %A, i64 %2 + %3 = load i32* %arrayidx2, align 4 + %add3 = add nsw i32 %3, %1 + %arrayidx5 = getelementptr inbounds i32* %A, i64 %indvars.iv + store i32 %add3, i32* %arrayidx5, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} + +; Example: +; for (int i = ...; ++i) { +; a[i] = b[i]; +; c[i] = a[i-3] + ...; +; It is a bad idea to vectorize this loop because store-load forwarding will not +; happen. +; + +; CHECK-LABEL: @nostoreloadforward2( +; CHECK-NOT: <2 x i32> + +define void @nostoreloadforward2(i32* noalias %A, i32* noalias %B, i32* noalias %C) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 16, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds i32* %B, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds i32* %A, i64 %indvars.iv + store i32 %0, i32* %arrayidx2, align 4 + %1 = add nsw i64 %indvars.iv, -3 + %arrayidx4 = getelementptr inbounds i32* %A, i64 %1 + %2 = load i32* %arrayidx4, align 4 + %arrayidx6 = getelementptr inbounds i32* %C, i64 %indvars.iv + store i32 %2, i32* %arrayidx6, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/metadata-unroll.ll b/test/Transforms/LoopVectorize/metadata-unroll.ll new file mode 100644 index 0000000000000..7f1037200659e --- /dev/null +++ b/test/Transforms/LoopVectorize/metadata-unroll.ll @@ -0,0 +1,41 @@ +; RUN: opt < %s -loop-vectorize -force-vector-width=4 -dce -instcombine -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +@a = common global [2048 x i32] zeroinitializer, align 16 + +; This is the loop. +; for (i=0; i<n; i++){ +; a[i] += i; +; } +;CHECK-LABEL: @inc( +;CHECK: load <4 x i32> +;CHECK: load <4 x i32> +;CHECK: add nsw <4 x i32> +;CHECK: add nsw <4 x i32> +;CHECK: store <4 x i32> +;CHECK: store <4 x i32> +;CHECK: ret void +define void @inc(i32 %n) nounwind uwtable noinline ssp { + %1 = icmp sgt i32 %n, 0 + br i1 %1, label %.lr.ph, label %._crit_edge + +.lr.ph: ; preds = %0, %.lr.ph + %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ] + %2 = getelementptr inbounds [2048 x i32]* @a, i64 0, i64 %indvars.iv + %3 = load i32* %2, align 4 + %4 = trunc i64 %indvars.iv to i32 + %5 = add nsw i32 %3, %4 + store i32 %5, i32* %2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %._crit_edge, label %.lr.ph, !llvm.loop !0 + +._crit_edge: ; preds = %.lr.ph, %0 + ret void +} + +!0 = metadata !{metadata !0, metadata !1} +!1 = metadata !{metadata !"llvm.vectorizer.unroll", i32 2} diff --git a/test/Transforms/LoopVectorize/metadata-width.ll b/test/Transforms/LoopVectorize/metadata-width.ll new file mode 100644 index 0000000000000..1960c0bad6bc1 --- /dev/null +++ b/test/Transforms/LoopVectorize/metadata-width.ll @@ -0,0 +1,31 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -dce -instcombine -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK-LABEL: @test1( +; CHECK: store <8 x i32> +; CHECK: ret void +define void @test1(i32* nocapture %a, i32 %n) #0 { +entry: + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = trunc i64 %indvars.iv to i32 + store i32 %0, i32* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0 + +for.end: ; preds = %for.body, %entry + ret void +} + +attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!0 = metadata !{metadata !0, metadata !1} +!1 = metadata !{metadata !"llvm.vectorizer.width", i32 8} diff --git a/test/Transforms/LoopVectorize/minmax_reduction.ll b/test/Transforms/LoopVectorize/minmax_reduction.ll index 502fd8b9383b1..0e47260984f22 100644 --- a/test/Transforms/LoopVectorize/minmax_reduction.ll +++ b/test/Transforms/LoopVectorize/minmax_reduction.ll @@ -10,14 +10,14 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; Turn this into a max reduction. Make sure we use a splat to initialize the ; vector for the reduction. -; CHECK: @max_red +; CHECK-LABEL: @max_red( ; CHECK: %[[VAR:.*]] = insertelement <2 x i32> undef, i32 %max, i32 0 ; CHECK: {{.*}} = shufflevector <2 x i32> %[[VAR]], <2 x i32> undef, <2 x i32> zeroinitializer ; CHECK: icmp sgt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp sgt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @max_red(i32 %max) { entry: @@ -41,12 +41,12 @@ for.end: ; Turn this into a max reduction. The select has its inputs reversed therefore ; this is a max reduction. -; CHECK: @max_red_inverse_select +; CHECK-LABEL: @max_red_inverse_select( ; CHECK: icmp slt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp sgt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @max_red_inverse_select(i32 %max) { entry: @@ -69,12 +69,12 @@ for.end: } ; Turn this into a min reduction. -; CHECK: @min_red +; CHECK-LABEL: @min_red( ; CHECK: icmp slt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp slt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @min_red(i32 %max) { entry: @@ -98,12 +98,12 @@ for.end: ; Turn this into a min reduction. The select has its inputs reversed therefore ; this is a min reduction. -; CHECK: @min_red_inverse_select +; CHECK-LABEL: @min_red_inverse_select( ; CHECK: icmp sgt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp slt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @min_red_inverse_select(i32 %max) { entry: @@ -128,12 +128,12 @@ for.end: ; Unsigned tests. ; Turn this into a max reduction. -; CHECK: @umax_red +; CHECK-LABEL: @umax_red( ; CHECK: icmp ugt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ugt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @umax_red(i32 %max) { entry: @@ -157,12 +157,12 @@ for.end: ; Turn this into a max reduction. The select has its inputs reversed therefore ; this is a max reduction. -; CHECK: @umax_red_inverse_select +; CHECK-LABEL: @umax_red_inverse_select( ; CHECK: icmp ult <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ugt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @umax_red_inverse_select(i32 %max) { entry: @@ -185,12 +185,12 @@ for.end: } ; Turn this into a min reduction. -; CHECK: @umin_red +; CHECK-LABEL: @umin_red( ; CHECK: icmp ult <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ult <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @umin_red(i32 %max) { entry: @@ -214,12 +214,12 @@ for.end: ; Turn this into a min reduction. The select has its inputs reversed therefore ; this is a min reduction. -; CHECK: @umin_red_inverse_select +; CHECK-LABEL: @umin_red_inverse_select( ; CHECK: icmp ugt <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ult <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @umin_red_inverse_select(i32 %max) { entry: @@ -243,12 +243,12 @@ for.end: ; SGE -> SLT ; Turn this into a min reduction (select inputs are reversed). -; CHECK: @sge_min_red +; CHECK-LABEL: @sge_min_red( ; CHECK: icmp sge <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp slt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @sge_min_red(i32 %max) { entry: @@ -272,12 +272,12 @@ for.end: ; SLE -> SGT ; Turn this into a max reduction (select inputs are reversed). -; CHECK: @sle_min_red +; CHECK-LABEL: @sle_min_red( ; CHECK: icmp sle <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp sgt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @sle_min_red(i32 %max) { entry: @@ -301,12 +301,12 @@ for.end: ; UGE -> ULT ; Turn this into a min reduction (select inputs are reversed). -; CHECK: @uge_min_red +; CHECK-LABEL: @uge_min_red( ; CHECK: icmp uge <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ult <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @uge_min_red(i32 %max) { entry: @@ -330,12 +330,12 @@ for.end: ; ULE -> UGT ; Turn this into a max reduction (select inputs are reversed). -; CHECK: @ule_min_red +; CHECK-LABEL: @ule_min_red( ; CHECK: icmp ule <2 x i32> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: icmp ugt <2 x i32> -; CHECK: select <2 x i1> +; CHECK: select i1 define i32 @ule_min_red(i32 %max) { entry: @@ -358,7 +358,7 @@ for.end: } ; No reduction. -; CHECK: @no_red_1 +; CHECK-LABEL: @no_red_1( ; CHECK-NOT: icmp <2 x i32> define i32 @no_red_1(i32 %max) { entry: @@ -382,7 +382,7 @@ for.end: ret i32 %max.red.0 } -; CHECK: @no_red_2 +; CHECK-LABEL: @no_red_2( ; CHECK-NOT: icmp <2 x i32> define i32 @no_red_2(i32 %max) { entry: @@ -411,12 +411,12 @@ for.end: ; Maximum. ; Turn this into a max reduction in the presence of a no-nans-fp-math attribute. -; CHECK: @max_red_float +; CHECK-LABEL: @max_red_float( ; CHECK: fcmp ogt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @max_red_float(float %max) #0 { entry: @@ -437,12 +437,12 @@ for.end: ret float %max.red.0 } -; CHECK: @max_red_float_ge +; CHECK-LABEL: @max_red_float_ge( ; CHECK: fcmp oge <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @max_red_float_ge(float %max) #0 { entry: @@ -463,12 +463,12 @@ for.end: ret float %max.red.0 } -; CHECK: @inverted_max_red_float +; CHECK-LABEL: @inverted_max_red_float( ; CHECK: fcmp olt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_max_red_float(float %max) #0 { entry: @@ -489,12 +489,12 @@ for.end: ret float %max.red.0 } -; CHECK: @inverted_max_red_float_le +; CHECK-LABEL: @inverted_max_red_float_le( ; CHECK: fcmp ole <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_max_red_float_le(float %max) #0 { entry: @@ -515,12 +515,12 @@ for.end: ret float %max.red.0 } -; CHECK: @unordered_max_red +; CHECK-LABEL: @unordered_max_red_float( ; CHECK: fcmp ugt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @unordered_max_red_float(float %max) #0 { entry: @@ -541,12 +541,12 @@ for.end: ret float %max.red.0 } -; CHECK: @unordered_max_red_float_ge +; CHECK-LABEL: @unordered_max_red_float_ge( ; CHECK: fcmp uge <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @unordered_max_red_float_ge(float %max) #0 { entry: @@ -567,12 +567,12 @@ for.end: ret float %max.red.0 } -; CHECK: @inverted_unordered_max_red +; CHECK-LABEL: @inverted_unordered_max_red_float( ; CHECK: fcmp ult <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_unordered_max_red_float(float %max) #0 { entry: @@ -593,12 +593,12 @@ for.end: ret float %max.red.0 } -; CHECK: @inverted_unordered_max_red_float_le +; CHECK-LABEL: @inverted_unordered_max_red_float_le( ; CHECK: fcmp ule <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp ogt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_unordered_max_red_float_le(float %max) #0 { entry: @@ -622,12 +622,12 @@ for.end: ; Minimum. ; Turn this into a min reduction in the presence of a no-nans-fp-math attribute. -; CHECK: @min_red_float +; CHECK-LABEL: @min_red_float( ; CHECK: fcmp olt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @min_red_float(float %min) #0 { entry: @@ -648,12 +648,12 @@ for.end: ret float %min.red.0 } -; CHECK: @min_red_float_le +; CHECK-LABEL: @min_red_float_le( ; CHECK: fcmp ole <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @min_red_float_le(float %min) #0 { entry: @@ -674,12 +674,12 @@ for.end: ret float %min.red.0 } -; CHECK: @inverted_min_red_float +; CHECK-LABEL: @inverted_min_red_float( ; CHECK: fcmp ogt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_min_red_float(float %min) #0 { entry: @@ -700,12 +700,12 @@ for.end: ret float %min.red.0 } -; CHECK: @inverted_min_red_float_ge +; CHECK-LABEL: @inverted_min_red_float_ge( ; CHECK: fcmp oge <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_min_red_float_ge(float %min) #0 { entry: @@ -726,12 +726,12 @@ for.end: ret float %min.red.0 } -; CHECK: @unordered_min_red +; CHECK-LABEL: @unordered_min_red_float( ; CHECK: fcmp ult <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @unordered_min_red_float(float %min) #0 { entry: @@ -752,12 +752,12 @@ for.end: ret float %min.red.0 } -; CHECK: @unordered_min_red_float_le +; CHECK-LABEL: @unordered_min_red_float_le( ; CHECK: fcmp ule <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @unordered_min_red_float_le(float %min) #0 { entry: @@ -778,12 +778,12 @@ for.end: ret float %min.red.0 } -; CHECK: @inverted_unordered_min_red +; CHECK-LABEL: @inverted_unordered_min_red_float( ; CHECK: fcmp ugt <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_unordered_min_red_float(float %min) #0 { entry: @@ -804,12 +804,12 @@ for.end: ret float %min.red.0 } -; CHECK: @inverted_unordered_min_red_float_ge +; CHECK-LABEL: @inverted_unordered_min_red_float_ge( ; CHECK: fcmp uge <2 x float> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x float> -; CHECK: select <2 x i1> +; CHECK: select i1 define float @inverted_unordered_min_red_float_ge(float %min) #0 { entry: @@ -831,12 +831,12 @@ for.end: } ; Make sure we handle doubles, too. -; CHECK: @min_red_double +; CHECK-LABEL: @min_red_double( ; CHECK: fcmp olt <2 x double> ; CHECK: select <2 x i1> ; CHECK: middle.block ; CHECK: fcmp olt <2 x double> -; CHECK: select <2 x i1> +; CHECK: select i1 define double @min_red_double(double %min) #0 { entry: @@ -859,7 +859,7 @@ for.end: ; Don't this into a max reduction. The no-nans-fp-math attribute is missing -; CHECK: @max_red_float_nans +; CHECK-LABEL: @max_red_float_nans( ; CHECK-NOT: <2 x float> define float @max_red_float_nans(float %max) { @@ -882,4 +882,4 @@ for.end: } -attributes #0 = { "no-nans-fp-math"="true" } +attributes #0 = { "no-nans-fp-math"="true" } diff --git a/test/Transforms/LoopVectorize/multiple-address-spaces.ll b/test/Transforms/LoopVectorize/multiple-address-spaces.ll new file mode 100644 index 0000000000000..7d836dedbdbb2 --- /dev/null +++ b/test/Transforms/LoopVectorize/multiple-address-spaces.ll @@ -0,0 +1,44 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s + +; From a simple program with two address spaces: +; char Y[4*10000] __attribute__((address_space(1))); +; char X[4*10000]; +; int main() { +; for (int i = 0; i < 4*10000; ++i) +; X[i] = Y[i] + 1; +; return 0; +;} + + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@Y = common addrspace(1) global [40000 x i8] zeroinitializer, align 16 +@X = common global [40000 x i8] zeroinitializer, align 16 + +;CHECK-LABEL: @main( +;CHECK: bitcast i8 addrspace(1)* %{{.*}} to <4 x i8> addrspace(1)* +;CHECK: bitcast i8* %{{.*}} to <4 x i8>* + +; Function Attrs: nounwind uwtable +define i32 @main() #0 { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %arrayidx = getelementptr inbounds [40000 x i8] addrspace(1)* @Y, i64 0, i64 %indvars.iv + %0 = load i8 addrspace(1)* %arrayidx, align 1 + %add = add i8 %0, 1 + %arrayidx3 = getelementptr inbounds [40000 x i8]* @X, i64 0, i64 %indvars.iv + store i8 %add, i8* %arrayidx3, align 1 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, 40000 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret i32 0 +} + +attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-frame-pointer-elim-non-leaf"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/test/Transforms/LoopVectorize/no_idiv_reduction.ll b/test/Transforms/LoopVectorize/no_idiv_reduction.ll index cdfb3fd66f054..295fcabb0b291 100644 --- a/test/Transforms/LoopVectorize/no_idiv_reduction.ll +++ b/test/Transforms/LoopVectorize/no_idiv_reduction.ll @@ -7,7 +7,7 @@ entry: br label %for.body for.body: - ; CHECK: @g + ; CHECK-LABEL: @g( ; CHECK-NOT: sdiv <2 x i32> %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] %r.05 = phi i32 [ 80, %entry ], [ %div, %for.body ] diff --git a/test/Transforms/LoopVectorize/no_int_induction.ll b/test/Transforms/LoopVectorize/no_int_induction.ll index 45aa8c7cd9be5..e572d1a884cad 100644 --- a/test/Transforms/LoopVectorize/no_int_induction.ll +++ b/test/Transforms/LoopVectorize/no_int_induction.ll @@ -4,10 +4,10 @@ ; return std::accumulate(A, A + n, 0); ; } -target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" -target triple = "x86_64-apple-macosx10.8.0" +target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128" -;CHECK: @sum_array +;CHECK-LABEL: @sum_array( +;CHECK: phi i64 ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> @@ -31,3 +31,30 @@ _ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %.lr.ph.i, %0 %.01.lcssa.i = phi i32 [ 0, %0 ], [ %5, %.lr.ph.i ] ret i32 %.01.lcssa.i } + +; Same, but use a pointer with a different size. +;CHECK-LABEL: @sum_array_as1( +;CHECK: phi i16 +;CHECK: phi <4 x i32> +;CHECK: load <4 x i32> +;CHECK: add nsw <4 x i32> +;CHECK: ret i32 +define i32 @sum_array_as1(i32 addrspace(1)* %A, i32 %n) nounwind uwtable readonly noinline ssp { + %1 = sext i32 %n to i64 + %2 = getelementptr inbounds i32 addrspace(1)* %A, i64 %1 + %3 = icmp eq i32 %n, 0 + br i1 %3, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i + +.lr.ph.i: ; preds = %0, %.lr.ph.i + %.03.i = phi i32 addrspace(1)* [ %6, %.lr.ph.i ], [ %A, %0 ] + %.012.i = phi i32 [ %5, %.lr.ph.i ], [ 0, %0 ] + %4 = load i32 addrspace(1)* %.03.i, align 4 + %5 = add nsw i32 %4, %.012.i + %6 = getelementptr inbounds i32 addrspace(1)* %.03.i, i64 1 + %7 = icmp eq i32 addrspace(1)* %6, %2 + br i1 %7, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %.lr.ph.i + +_ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %.lr.ph.i, %0 + %.01.lcssa.i = phi i32 [ 0, %0 ], [ %5, %.lr.ph.i ] + ret i32 %.01.lcssa.i +} diff --git a/test/Transforms/LoopVectorize/no_outside_user.ll b/test/Transforms/LoopVectorize/no_outside_user.ll index 6f0357c5e546f..1f891ad2c4fa1 100644 --- a/test/Transforms/LoopVectorize/no_outside_user.ll +++ b/test/Transforms/LoopVectorize/no_outside_user.ll @@ -12,6 +12,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3 ; We used to vectorize this loop. But it has a value that is used outside of the ; and is not a recognized reduction variable "tmp17". +; CHECK-LABEL: @main( ; CHECK-NOT: <2 x i32> define i32 @main() { @@ -38,4 +39,33 @@ f1.exit.loopexit: ret i32 %.lcssa } +; Don't vectorize this loop. Its phi node (induction variable) has an outside +; loop user. We currently don't handle this case. +; PR17179 +; CHECK-LABEL: @test2( +; CHECK-NOT: <2 x + +@x1 = common global i32 0, align 4 +@x2 = common global i32 0, align 4 +@x0 = common global i32 0, align 4 + +define i32 @test2() { +entry: + store i32 0, i32* @x1, align 4 + %0 = load i32* @x0, align 4 + br label %for.cond1.preheader + +for.cond1.preheader: + %inc7 = phi i32 [ 0, %entry ], [ %inc, %for.cond1.preheader ] + %inc = add nsw i32 %inc7, 1 + %cmp = icmp eq i32 %inc, 52 + br i1 %cmp, label %for.end5, label %for.cond1.preheader + +for.end5: + %inc7.lcssa = phi i32 [ %inc7, %for.cond1.preheader ] + %xor = xor i32 %inc7.lcssa, %0 + store i32 52, i32* @x1, align 4 + store i32 1, i32* @x2, align 4 + ret i32 %xor +} diff --git a/test/Transforms/LoopVectorize/nofloat.ll b/test/Transforms/LoopVectorize/nofloat.ll index de23bf02b63aa..c3c81b6f84509 100644 --- a/test/Transforms/LoopVectorize/nofloat.ll +++ b/test/Transforms/LoopVectorize/nofloat.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0" @a = common global [2048 x i32] zeroinitializer, align 16 -;CHECK: @example12 +;CHECK-LABEL: @example12( ;CHECK-NOT: store <4 x i32> ;CHECK: ret void define void @example12() noimplicitfloat { ; <--------- "noimplicitfloat" attribute here! diff --git a/test/Transforms/LoopVectorize/non-const-n.ll b/test/Transforms/LoopVectorize/non-const-n.ll index 8262a18f1807e..0c54a2b01226a 100644 --- a/test/Transforms/LoopVectorize/non-const-n.ll +++ b/test/Transforms/LoopVectorize/non-const-n.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0" @c = common global [2048 x i32] zeroinitializer, align 16 @a = common global [2048 x i32] zeroinitializer, align 16 -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: shl i32 ;CHECK: zext i32 ;CHECK: load <4 x i32> diff --git a/test/Transforms/LoopVectorize/opt.ll b/test/Transforms/LoopVectorize/opt.ll new file mode 100644 index 0000000000000..27030a2ff2a97 --- /dev/null +++ b/test/Transforms/LoopVectorize/opt.ll @@ -0,0 +1,28 @@ +; RUN: opt -S -O3 -force-vector-width=2 -force-vector-unroll=1 < %s | FileCheck --check-prefix=LOOPVEC %s +; RUN: opt -S -O3 -disable-loop-vectorization -force-vector-width=2 -force-vector-unroll=1 < %s | FileCheck --check-prefix=NOLOOPVEC %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +; Make sure we can disable vectorization in opt. + +; LOOPVEC: add <2 x i32> +; NOLOOPVEC-NOT: add <2 x i32> + +define i32 @vect(i32* %a) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %red.05 = phi i32 [ 0, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds i32* %a, i64 %indvars.iv + %0 = load i32* %arrayidx, align 4 + %add = add nsw i32 %0, %red.05 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %exitcond = icmp eq i64 %indvars.iv.next, 255 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret i32 %add +} diff --git a/test/Transforms/LoopVectorize/ptr_loops.ll b/test/Transforms/LoopVectorize/ptr_loops.ll index 25599f8f4c3ca..15983f068556f 100644 --- a/test/Transforms/LoopVectorize/ptr_loops.ll +++ b/test/Transforms/LoopVectorize/ptr_loops.ll @@ -6,7 +6,7 @@ target triple = "x86_64-apple-macosx10.8.0" @A = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16 @B = global [36 x i32] [i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35], align 16 -;CHECK:_Z5test1v +;CHECK-LABEL:@_Z5test1v( ;CHECK: load <4 x i32> ;CHECK: shufflevector <4 x i32> ;CHECK: store <4 x i32> @@ -29,7 +29,7 @@ define i32 @_Z5test1v() nounwind uwtable ssp { ret i32 0 } -;CHECK:_Z5test2v +;CHECK-LABEL: @_Z5test2v( ;CHECK: load <4 x i32> ;CHECK: shufflevector <4 x i32> ;CHECK: store <4 x i32> diff --git a/test/Transforms/LoopVectorize/read-only.ll b/test/Transforms/LoopVectorize/read-only.ll index bfaa6d452bcee..fc8f0a5482f0b 100644 --- a/test/Transforms/LoopVectorize/read-only.ll +++ b/test/Transforms/LoopVectorize/read-only.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @read_only_func +;CHECK-LABEL: @read_only_func( ;CHECK: load <4 x i32> ;CHECK: ret i32 define i32 @read_only_func(i32* nocapture %A, i32* nocapture %B, i32 %n) nounwind uwtable readonly ssp { diff --git a/test/Transforms/LoopVectorize/reduction.ll b/test/Transforms/LoopVectorize/reduction.ll index 08b7b27e4257e..791fce1562202 100644 --- a/test/Transforms/LoopVectorize/reduction.ll +++ b/test/Transforms/LoopVectorize/reduction.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @reduction_sum +;CHECK-LABEL: @reduction_sum( ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: add <4 x i32> @@ -38,7 +38,7 @@ define i32 @reduction_sum(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ret i32 %sum.0.lcssa } -;CHECK: @reduction_prod +;CHECK-LABEL: @reduction_prod( ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: mul <4 x i32> @@ -73,7 +73,7 @@ define i32 @reduction_prod(i32 %n, i32* noalias nocapture %A, i32* noalias nocap ret i32 %prod.0.lcssa } -;CHECK: @reduction_mix +;CHECK-LABEL: @reduction_mix( ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: mul nsw <4 x i32> @@ -108,7 +108,7 @@ define i32 @reduction_mix(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ret i32 %sum.0.lcssa } -;CHECK: @reduction_mul +;CHECK-LABEL: @reduction_mul( ;CHECK: mul <4 x i32> ;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> ;CHECK: mul <4 x i32> @@ -141,7 +141,7 @@ define i32 @reduction_mul(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ret i32 %sum.0.lcssa } -;CHECK: @start_at_non_zero +;CHECK-LABEL: @start_at_non_zero( ;CHECK: phi <4 x i32> ;CHECK: <i32 120, i32 0, i32 0, i32 0> ;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> @@ -174,7 +174,7 @@ for.end: ; preds = %for.body, %entry ret i32 %sum.0.lcssa } -;CHECK: @reduction_and +;CHECK-LABEL: @reduction_and( ;CHECK: and <4 x i32> ;CHECK: <i32 -1, i32 -1, i32 -1, i32 -1> ;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> @@ -207,7 +207,7 @@ for.end: ; preds = %for.body, %entry ret i32 %result.0.lcssa } -;CHECK: @reduction_or +;CHECK-LABEL: @reduction_or( ;CHECK: or <4 x i32> ;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> ;CHECK: or <4 x i32> @@ -239,7 +239,7 @@ for.end: ; preds = %for.body, %entry ret i32 %result.0.lcssa } -;CHECK: @reduction_xor +;CHECK-LABEL: @reduction_xor( ;CHECK: xor <4 x i32> ;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef> ;CHECK: xor <4 x i32> @@ -272,7 +272,7 @@ for.end: ; preds = %for.body, %entry } ; In this code the subtracted variable is on the RHS and this is not an induction variable. -;CHECK: @reduction_sub_rhs +;CHECK-LABEL: @reduction_sub_rhs( ;CHECK-NOT: phi <4 x i32> ;CHECK-NOT: sub nsw <4 x i32> ;CHECK: ret i32 @@ -299,7 +299,7 @@ for.end: ; preds = %for.body, %entry ; In this test the reduction variable is on the LHS and we can vectorize it. -;CHECK: @reduction_sub_lhs +;CHECK-LABEL: @reduction_sub_lhs( ;CHECK: phi <4 x i32> ;CHECK: sub nsw <4 x i32> ;CHECK: ret i32 @@ -323,3 +323,174 @@ for.end: ; preds = %for.body, %entry %x.0.lcssa = phi i32 [ 0, %entry ], [ %sub, %for.body ] ret i32 %x.0.lcssa } + +; We can vectorize conditional reductions with multi-input phis. +; CHECK: reduction_conditional +; CHECK: fadd <4 x float> + +define float @reduction_conditional(float* %A, float* %B, float* %C, float %S) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ] + %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds float* %A, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds float* %B, i64 %indvars.iv + %1 = load float* %arrayidx2, align 4 + %cmp3 = fcmp ogt float %0, %1 + br i1 %cmp3, label %if.then, label %for.inc + +if.then: + %cmp6 = fcmp ogt float %1, 1.000000e+00 + br i1 %cmp6, label %if.then8, label %if.else + +if.then8: + %add = fadd fast float %sum.033, %0 + br label %for.inc + +if.else: + %cmp14 = fcmp ogt float %0, 2.000000e+00 + br i1 %cmp14, label %if.then16, label %for.inc + +if.then16: + %add19 = fadd fast float %sum.033, %1 + br label %for.inc + +for.inc: + %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ %sum.033, %if.else ], [ %sum.033, %for.body ] + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + %sum.1.lcssa = phi float [ %sum.1, %for.inc ] + ret float %sum.1.lcssa +} + +; We can't vectorize reductions with phi inputs from outside the reduction. +; CHECK: noreduction_phi +; CHECK-NOT: fadd <4 x float> +define float @noreduction_phi(float* %A, float* %B, float* %C, float %S) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ] + %sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ] + %arrayidx = getelementptr inbounds float* %A, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %arrayidx2 = getelementptr inbounds float* %B, i64 %indvars.iv + %1 = load float* %arrayidx2, align 4 + %cmp3 = fcmp ogt float %0, %1 + br i1 %cmp3, label %if.then, label %for.inc + +if.then: + %cmp6 = fcmp ogt float %1, 1.000000e+00 + br i1 %cmp6, label %if.then8, label %if.else + +if.then8: + %add = fadd fast float %sum.033, %0 + br label %for.inc + +if.else: + %cmp14 = fcmp ogt float %0, 2.000000e+00 + br i1 %cmp14, label %if.then16, label %for.inc + +if.then16: + %add19 = fadd fast float %sum.033, %1 + br label %for.inc + +for.inc: + %sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ 0.000000e+00, %if.else ], [ %sum.033, %for.body ] + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + %sum.1.lcssa = phi float [ %sum.1, %for.inc ] + ret float %sum.1.lcssa +} + +; We can't vectorize reductions that feed another header PHI. +; CHECK: noredux_header_phi +; CHECK-NOT: fadd <4 x float> + +define float @noredux_header_phi(float* %A, float* %B, float* %C, float %S) { +entry: + br label %for.body + +for.body: + %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] + %sum2.09 = phi float [ 0.000000e+00, %entry ], [ %add1, %for.body ] + %sum.08 = phi float [ %S, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds float* %B, i64 %indvars.iv + %0 = load float* %arrayidx, align 4 + %add = fadd fast float %sum.08, %0 + %add1 = fadd fast float %sum2.09, %add + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp ne i32 %lftr.wideiv, 128 + br i1 %exitcond, label %for.body, label %for.end + +for.end: + %add1.lcssa = phi float [ %add1, %for.body ] + %add.lcssa = phi float [ %add, %for.body ] + %add2 = fadd fast float %add.lcssa, %add1.lcssa + ret float %add2 +} + + +; When vectorizing a reduction whose loop header phi value is used outside the +; loop special care must be taken. Otherwise, the reduced value feeding into the +; outside user misses a few iterations (VF-1) of the loop. +; PR16522 + +; CHECK-LABEL: @phivalueredux( +; CHECK-NOT: x i32> + +define i32 @phivalueredux(i32 %p) { +entry: + br label %for.body + +for.body: + %t.03 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %p.addr.02 = phi i32 [ %p, %entry ], [ %xor, %for.body ] + %xor = xor i32 %p.addr.02, -1 + %inc = add nsw i32 %t.03, 1 + %exitcond = icmp eq i32 %inc, 16 + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret i32 %p.addr.02 +} + +; Don't vectorize a reduction value that is not the last in a reduction cyle. We +; would loose iterations (VF-1) on the operations after that use. +; PR17498 + +; CHECK-LABEL: not_last_operation +; CHECK-NOT: x i32> +define i32 @not_last_operation(i32 %p, i32 %val) { +entry: + %tobool = icmp eq i32 %p, 0 + br label %for.body + +for.body: + %inc613.1 = phi i32 [ 0, %entry ], [ %inc6.1, %for.body ] + %inc511.1 = phi i32 [ %val, %entry ], [ %inc5.1, %for.body ] + %0 = zext i1 %tobool to i32 + %inc4.1 = xor i32 %0, 1 + %inc511.1.inc4.1 = add nsw i32 %inc511.1, %inc4.1 + %inc5.1 = add nsw i32 %inc511.1.inc4.1, 1 + %inc6.1 = add nsw i32 %inc613.1, 1 + %exitcond.1 = icmp eq i32 %inc6.1, 22 + br i1 %exitcond.1, label %exit, label %for.body + +exit: + %inc.2 = add nsw i32 %inc511.1.inc4.1, 2 + ret i32 %inc.2 +} diff --git a/test/Transforms/LoopVectorize/reverse_induction.ll b/test/Transforms/LoopVectorize/reverse_induction.ll index f43f02bc3132a..65ef95dcb1210 100644 --- a/test/Transforms/LoopVectorize/reverse_induction.ll +++ b/test/Transforms/LoopVectorize/reverse_induction.ll @@ -5,7 +5,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; Make sure consecutive vector generates correct negative indices. ; PR15882 -; CHECK: reverse_induction_i64 +; CHECK-LABEL: @reverse_induction_i64( ; CHECK: add <4 x i64> %[[SPLAT:.*]], <i64 0, i64 -1, i64 -2, i64 -3> ; CHECK: add <4 x i64> %[[SPLAT]], <i64 -4, i64 -5, i64 -6, i64 -7> @@ -29,7 +29,7 @@ loopend: ret i32 %inc.redux } -; CHECK: reverse_induction_i128 +; CHECK-LABEL: @reverse_induction_i128( ; CHECK: add <4 x i128> %[[SPLAT:.*]], <i128 0, i128 -1, i128 -2, i128 -3> ; CHECK: add <4 x i128> %[[SPLAT]], <i128 -4, i128 -5, i128 -6, i128 -7> define i32 @reverse_induction_i128(i128 %startval, i32 * %ptr) { @@ -52,7 +52,7 @@ loopend: ret i32 %inc.redux } -; CHECK: reverse_induction_i16 +; CHECK-LABEL: @reverse_induction_i16( ; CHECK: add <4 x i16> %[[SPLAT:.*]], <i16 0, i16 -1, i16 -2, i16 -3> ; CHECK: add <4 x i16> %[[SPLAT]], <i16 -4, i16 -5, i16 -6, i16 -7> @@ -77,3 +77,72 @@ loopend: } +@a = common global [1024 x i32] zeroinitializer, align 16 + +; We incorrectly transformed this loop into an empty one because we left the +; induction variable in i8 type and truncated the exit value 1024 to 0. +; int a[1024]; +; +; void fail() { +; int reverse_induction = 1023; +; unsigned char forward_induction = 0; +; while ((reverse_induction) >= 0) { +; forward_induction++; +; a[reverse_induction] = forward_induction; +; --reverse_induction; +; } +; } + +; CHECK-LABEL: @reverse_forward_induction_i64_i8( +; CHECK: vector.body +; CHECK: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK: %normalized.idx = sub i64 %index, 0 +; CHECK: %reverse.idx = sub i64 1023, %normalized.idx +; CHECK: trunc i64 %index to i8 + +define void @reverse_forward_induction_i64_i8() { +entry: + br label %while.body + +while.body: + %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %while.body ] + %forward_induction.05 = phi i8 [ 0, %entry ], [ %inc, %while.body ] + %inc = add i8 %forward_induction.05, 1 + %conv = zext i8 %inc to i32 + %arrayidx = getelementptr inbounds [1024 x i32]* @a, i64 0, i64 %indvars.iv + store i32 %conv, i32* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, -1 + %0 = trunc i64 %indvars.iv to i32 + %cmp = icmp sgt i32 %0, 0 + br i1 %cmp, label %while.body, label %while.end + +while.end: + ret void +} + +; CHECK-LABEL: @reverse_forward_induction_i64_i8_signed( +; CHECK: vector.body: +; CHECK: %index = phi i64 [ 129, %vector.ph ], [ %index.next, %vector.body ] +; CHECK: %normalized.idx = sub i64 %index, 129 +; CHECK: %reverse.idx = sub i64 1023, %normalized.idx +; CHECK: trunc i64 %index to i8 + +define void @reverse_forward_induction_i64_i8_signed() { +entry: + br label %while.body + +while.body: + %indvars.iv = phi i64 [ 1023, %entry ], [ %indvars.iv.next, %while.body ] + %forward_induction.05 = phi i8 [ -127, %entry ], [ %inc, %while.body ] + %inc = add i8 %forward_induction.05, 1 + %conv = sext i8 %inc to i32 + %arrayidx = getelementptr inbounds [1024 x i32]* @a, i64 0, i64 %indvars.iv + store i32 %conv, i32* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, -1 + %0 = trunc i64 %indvars.iv to i32 + %cmp = icmp sgt i32 %0, 0 + br i1 %cmp, label %while.body, label %while.end + +while.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/reverse_iter.ll b/test/Transforms/LoopVectorize/reverse_iter.ll new file mode 100644 index 0000000000000..f803120c4d570 --- /dev/null +++ b/test/Transforms/LoopVectorize/reverse_iter.ll @@ -0,0 +1,45 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +; Make sure that the reverse iterators are calculated using 64bit arithmetic, not 32. +; +; int foo(int n, int *A) { +; int sum; +; for (int i=n; i > 0; i--) +; sum += A[i*2]; +; return sum; +; } +; + +;CHECK-LABEL: @foo( +;CHECK: <i64 0, i64 -1, i64 -2, i64 -3> +;CHECK: ret +define i32 @foo(i32 %n, i32* nocapture %A) { + %1 = icmp sgt i32 %n, 0 + br i1 %1, label %.lr.ph, label %._crit_edge + +.lr.ph: ; preds = %0 + %2 = sext i32 %n to i64 + br label %3 + +; <label>:3 ; preds = %.lr.ph, %3 + %indvars.iv = phi i64 [ %2, %.lr.ph ], [ %indvars.iv.next, %3 ] + %sum.01 = phi i32 [ undef, %.lr.ph ], [ %9, %3 ] + %4 = trunc i64 %indvars.iv to i32 + %5 = shl nsw i32 %4, 1 + %6 = sext i32 %5 to i64 + %7 = getelementptr inbounds i32* %A, i64 %6 + %8 = load i32* %7, align 4 + %9 = add nsw i32 %8, %sum.01 + %indvars.iv.next = add i64 %indvars.iv, -1 + %10 = trunc i64 %indvars.iv.next to i32 + %11 = icmp sgt i32 %10, 0 + br i1 %11, label %3, label %._crit_edge + +._crit_edge: ; preds = %3, %0 + %sum.0.lcssa = phi i32 [ undef, %0 ], [ %9, %3 ] + ret i32 %sum.0.lcssa +} + diff --git a/test/Transforms/LoopVectorize/runtime-check-address-space.ll b/test/Transforms/LoopVectorize/runtime-check-address-space.ll new file mode 100644 index 0000000000000..6c86561a1c7e9 --- /dev/null +++ b/test/Transforms/LoopVectorize/runtime-check-address-space.ll @@ -0,0 +1,235 @@ +; RUN: opt -S -march=r600 -mcpu=cayman -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s + +; Check vectorization that would ordinarily require a runtime bounds +; check on the pointers when mixing address spaces. For now we cannot +; assume address spaces do not alias, and we can't assume that +; different pointers are directly comparable. +; +; These all test this basic loop for different combinations of address +; spaces, and swapping in globals or adding noalias. +; +;void foo(int addrspace(N)* [noalias] a, int addrspace(M)* [noalias] b, int n) +;{ +; for (int i = 0; i < n; ++i) +; { +; a[i] = 3 * b[i]; +; } +;} + +; Artificial datalayout +target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64" + + +@g_as1 = common addrspace(1) global [1024 x i32] zeroinitializer, align 16 +@q_as2 = common addrspace(2) global [1024 x i32] zeroinitializer, align 16 + +; Both parameters are unidentified objects with the same address +; space, so this should vectorize normally. +define void @foo(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %n) #0 { +; CHECK-LABEL: @foo( +; CHECK: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %idxprom + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds i32 addrspace(1)* %a, i64 %idxprom1 + store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +; Parameters are unidentified and different address spaces, so cannot vectorize. +define void @bar0(i32* %a, i32 addrspace(1)* %b, i32 %n) #0 { +; CHECK-LABEL: @bar0( +; CHECK-NOT: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %idxprom + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds i32* %a, i64 %idxprom1 + store i32 %mul, i32* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +; Swapped arguments should be the same +define void @bar1(i32 addrspace(1)* %a, i32* %b, i32 %n) #0 { +; CHECK-LABEL: @bar1( +; CHECK-NOT: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds i32* %b, i64 %idxprom + %0 = load i32* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds i32 addrspace(1)* %a, i64 %idxprom1 + store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +; We should still be able to vectorize with noalias even if the +; address spaces are different. +define void @bar2(i32* noalias %a, i32 addrspace(1)* noalias %b, i32 %n) #0 { +; CHECK-LABEL: @bar2( +; CHECK: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %idxprom + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds i32* %a, i64 %idxprom1 + store i32 %mul, i32* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +; Store to identified global with different address space. This isn't +; generally safe and shouldn't be vectorized. +define void @arst0(i32* %b, i32 %n) #0 { +; CHECK-LABEL: @arst0( +; CHECK-NOT: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds i32* %b, i64 %idxprom + %0 = load i32* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom1 + store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + + +; Load from identified global with different address space. +; This isn't generally safe and shouldn't be vectorized. +define void @arst1(i32* %b, i32 %n) #0 { +; CHECK-LABEL: @arst1( +; CHECK-NOT: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds i32* %b, i64 %idxprom1 + store i32 %mul, i32* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +; Read and write to 2 identified globals in different address +; spaces. This should be vectorized. +define void @aoeu(i32 %n) #0 { +; CHECK-LABEL: @aoeu( +; CHECK: <4 x i32> +; CHECK: ret + +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp slt i32 %i.0, %n + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %idxprom = sext i32 %i.0 to i64 + %arrayidx = getelementptr inbounds [1024 x i32] addrspace(2)* @q_as2, i64 0, i64 %idxprom + %0 = load i32 addrspace(2)* %arrayidx, align 4 + %mul = mul nsw i32 %0, 3 + %idxprom1 = sext i32 %i.0 to i64 + %arrayidx2 = getelementptr inbounds [1024 x i32] addrspace(1)* @g_as1, i64 0, i64 %idxprom1 + store i32 %mul, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add nsw i32 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll b/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll new file mode 100644 index 0000000000000..212b37cceab3a --- /dev/null +++ b/test/Transforms/LoopVectorize/runtime-check-readonly-address-space.ll @@ -0,0 +1,142 @@ +; RUN: opt -S -march=r600 -mcpu=cayman -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s + +; Artificial datalayout +target datalayout = "e-p:32:32:32-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64-v96:128:128-v128:128:128-v192:256:256-v256:256:256-v512:512:512-v1024:1024:1024-v2048:2048:2048-n32:64" + + +define void @add_ints_1_1_1(i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #0 { +; CHECK-LABEL: @add_ints_1_1_1( +; CHECK: <4 x i32> +; CHECK: ret +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp ult i64 %i.0, 200 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %i.0 + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32 addrspace(1)* %c, i64 %i.0 + %1 = load i32 addrspace(1)* %arrayidx1, align 4 + %add = add nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds i32 addrspace(1)* %a, i64 %i.0 + store i32 %add, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add i64 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +define void @add_ints_as_1_0_0(i32 addrspace(1)* %a, i32* %b, i32* %c) #0 { +; CHECK-LABEL: @add_ints_as_1_0_0( +; CHECK-NOT: <4 x i32> +; CHECK: ret +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp ult i64 %i.0, 200 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %arrayidx = getelementptr inbounds i32* %b, i64 %i.0 + %0 = load i32* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32* %c, i64 %i.0 + %1 = load i32* %arrayidx1, align 4 + %add = add nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds i32 addrspace(1)* %a, i64 %i.0 + store i32 %add, i32 addrspace(1)* %arrayidx2, align 4 + %inc = add i64 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +define void @add_ints_as_0_1_0(i32* %a, i32 addrspace(1)* %b, i32* %c) #0 { +; CHECK-LABEL: @add_ints_as_0_1_0( +; CHECK-NOT: <4 x i32> +; CHECK: ret +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp ult i64 %i.0, 200 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %i.0 + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32* %c, i64 %i.0 + %1 = load i32* %arrayidx1, align 4 + %add = add nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds i32* %a, i64 %i.0 + store i32 %add, i32* %arrayidx2, align 4 + %inc = add i64 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +define void @add_ints_as_0_1_1(i32* %a, i32 addrspace(1)* %b, i32 addrspace(1)* %c) #0 { +; CHECK-LABEL: @add_ints_as_0_1_1( +; CHECK-NOT: <4 x i32> +; CHECK: ret +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp ult i64 %i.0, 200 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %i.0 + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32 addrspace(1)* %c, i64 %i.0 + %1 = load i32 addrspace(1)* %arrayidx1, align 4 + %add = add nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds i32* %a, i64 %i.0 + store i32 %add, i32* %arrayidx2, align 4 + %inc = add i64 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +define void @add_ints_as_0_1_2(i32* %a, i32 addrspace(1)* %b, i32 addrspace(2)* %c) #0 { +; CHECK-LABEL: @add_ints_as_0_1_2( +; CHECK-NOT: <4 x i32> +; CHECK: ret +entry: + br label %for.cond + +for.cond: ; preds = %for.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %for.body ] + %cmp = icmp ult i64 %i.0, 200 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %arrayidx = getelementptr inbounds i32 addrspace(1)* %b, i64 %i.0 + %0 = load i32 addrspace(1)* %arrayidx, align 4 + %arrayidx1 = getelementptr inbounds i32 addrspace(2)* %c, i64 %i.0 + %1 = load i32 addrspace(2)* %arrayidx1, align 4 + %add = add nsw i32 %0, %1 + %arrayidx2 = getelementptr inbounds i32* %a, i64 %i.0 + store i32 %add, i32* %arrayidx2, align 4 + %inc = add i64 %i.0, 1 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} + +attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } diff --git a/test/Transforms/LoopVectorize/runtime-check-readonly.ll b/test/Transforms/LoopVectorize/runtime-check-readonly.ll index 4145d134fd70d..a2b9ad94c8376 100644 --- a/test/Transforms/LoopVectorize/runtime-check-readonly.ll +++ b/test/Transforms/LoopVectorize/runtime-check-readonly.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: add_ints +;CHECK-LABEL: @add_ints( ;CHECK: br ;CHECK: getelementptr ;CHECK-NEXT: getelementptr diff --git a/test/Transforms/LoopVectorize/runtime-check.ll b/test/Transforms/LoopVectorize/runtime-check.ll index 014c4fc48f877..d15479d202b7d 100644 --- a/test/Transforms/LoopVectorize/runtime-check.ll +++ b/test/Transforms/LoopVectorize/runtime-check.ll @@ -12,7 +12,7 @@ target triple = "x86_64-apple-macosx10.9.0" ;CHECK: for.body.preheader: ;CHECK: br i1 %cmp.zero, label %middle.block, label %vector.memcheck ;CHECK: vector.memcheck: -;CHECK: br i1 %found.conflict, label %middle.block, label %vector.ph +;CHECK: br i1 %memcheck.conflict, label %middle.block, label %vector.ph ;CHECK: load <4 x float> define i32 @foo(float* nocapture %a, float* nocapture %b, i32 %n) nounwind uwtable ssp { entry: @@ -34,3 +34,31 @@ for.body: ; preds = %entry, %for.body for.end: ; preds = %for.body, %entry ret i32 undef } + +; Make sure that we try to vectorize loops with a runtime check if the +; dependency check fails. + +; CHECK-LABEL: test_runtime_check +; CHECK: <4 x float> +define void @test_runtime_check(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n) { +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %ind.sum = add i64 %iv, %offset + %arr.idx = getelementptr inbounds float* %a, i64 %ind.sum + %l1 = load float* %arr.idx, align 4 + %ind.sum2 = add i64 %iv, %offset2 + %arr.idx2 = getelementptr inbounds float* %a, i64 %ind.sum2 + %l2 = load float* %arr.idx2, align 4 + %m = fmul fast float %b, %l2 + %ad = fadd fast float %l1, %m + store float %ad, float* %arr.idx, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %loopexit, label %for.body + +loopexit: + ret void +} diff --git a/test/Transforms/LoopVectorize/runtime-limit.ll b/test/Transforms/LoopVectorize/runtime-limit.ll index d7839746f0e15..7370a6fb93c56 100644 --- a/test/Transforms/LoopVectorize/runtime-limit.ll +++ b/test/Transforms/LoopVectorize/runtime-limit.ll @@ -4,7 +4,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 target triple = "x86_64-apple-macosx10.8.0" ; We are vectorizing with 6 runtime checks. -;CHECK: func1x6 +;CHECK-LABEL: func1x6( ;CHECK: <4 x i32> ;CHECK: ret define i32 @func1x6(i32* nocapture %out, i32* nocapture %A, i32* nocapture %B, i32* nocapture %C, i32* nocapture %D, i32* nocapture %E, i32* nocapture %F) { @@ -38,7 +38,7 @@ for.end: ; preds = %for.body } ; We are not vectorizing with 12 runtime checks. -;CHECK: func2x6 +;CHECK-LABEL: func2x6( ;CHECK-NOT: <4 x i32> ;CHECK: ret define i32 @func2x6(i32* nocapture %out, i32* nocapture %out2, i32* nocapture %A, i32* nocapture %B, i32* nocapture %C, i32* nocapture %D, i32* nocapture %E, i32* nocapture %F) { diff --git a/test/Transforms/LoopVectorize/safegep.ll b/test/Transforms/LoopVectorize/safegep.ll new file mode 100644 index 0000000000000..c9508601e2c1f --- /dev/null +++ b/test/Transforms/LoopVectorize/safegep.ll @@ -0,0 +1,61 @@ +; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-unroll=1 < %s | FileCheck %s +target datalayout = "e-p:32:32:32-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f16:16:16-f32:32:32-f64:32:64-f128:128:128-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32" + + +; We can vectorize this code because if the address computation would wrap then +; a load from 0 would take place which is undefined behaviour in address space 0 +; according to LLVM IR semantics. + +; PR16592 + +; CHECK-LABEL: @safe( +; CHECK: <4 x float> + +define void @safe(float* %A, float* %B, float %K) { +entry: + br label %"<bb 3>" + +"<bb 3>": + %i_15 = phi i32 [ 0, %entry ], [ %i_19, %"<bb 3>" ] + %pp3 = getelementptr float* %A, i32 %i_15 + %D.1396_10 = load float* %pp3, align 4 + %pp24 = getelementptr float* %B, i32 %i_15 + %D.1398_15 = load float* %pp24, align 4 + %D.1399_17 = fadd float %D.1398_15, %K + %D.1400_18 = fmul float %D.1396_10, %D.1399_17 + store float %D.1400_18, float* %pp3, align 4 + %i_19 = add nsw i32 %i_15, 1 + %exitcond = icmp ne i32 %i_19, 64 + br i1 %exitcond, label %"<bb 3>", label %return + +return: + ret void +} + +; In a non-default address space we don't have this rule. + +; CHECK-LABEL: @notsafe( +; CHECK-NOT: <4 x float> + +define void @notsafe(float addrspace(5) * %A, float* %B, float %K) { +entry: + br label %"<bb 3>" + +"<bb 3>": + %i_15 = phi i32 [ 0, %entry ], [ %i_19, %"<bb 3>" ] + %pp3 = getelementptr float addrspace(5) * %A, i32 %i_15 + %D.1396_10 = load float addrspace(5) * %pp3, align 4 + %pp24 = getelementptr float* %B, i32 %i_15 + %D.1398_15 = load float* %pp24, align 4 + %D.1399_17 = fadd float %D.1398_15, %K + %D.1400_18 = fmul float %D.1396_10, %D.1399_17 + store float %D.1400_18, float addrspace(5) * %pp3, align 4 + %i_19 = add nsw i32 %i_15, 1 + %exitcond = icmp ne i32 %i_19, 64 + br i1 %exitcond, label %"<bb 3>", label %return + +return: + ret void +} + + diff --git a/test/Transforms/LoopVectorize/same-base-access.ll b/test/Transforms/LoopVectorize/same-base-access.ll index 15738936457a7..d623a34690966 100644 --- a/test/Transforms/LoopVectorize/same-base-access.ll +++ b/test/Transforms/LoopVectorize/same-base-access.ll @@ -11,7 +11,7 @@ target triple = "x86_64-apple-macosx10.9.0" ; x[k] = x[k-1] + y[k]; ; } -; CHECK: @kernel11 +; CHECK-LABEL: @kernel11( ; CHECK-NOT: <4 x double> ; CHECK: ret define i32 @kernel11(double* %x, double* %y, i32 %n) nounwind uwtable ssp { @@ -77,7 +77,7 @@ define i32 @kernel11(double* %x, double* %y, i32 %n) nounwind uwtable ssp { ; } ; } -; CHECK: @func2 +; CHECK-LABEL: @func2( ; CHECK-NOT: <4 x i32> ; CHECK: ret define i32 @func2(i32* nocapture %a) nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/scalar-select.ll b/test/Transforms/LoopVectorize/scalar-select.ll index 7a14d247c9b43..257c7bebe4d54 100644 --- a/test/Transforms/LoopVectorize/scalar-select.ll +++ b/test/Transforms/LoopVectorize/scalar-select.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0" @b = common global [2048 x i32] zeroinitializer, align 16 @c = common global [2048 x i32] zeroinitializer, align 16 -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK: load <4 x i32> ; make sure that we have a scalar condition and a vector operand. ;CHECK: select i1 %cond, <4 x i32> diff --git a/test/Transforms/LoopVectorize/scev-exitlim-crash.ll b/test/Transforms/LoopVectorize/scev-exitlim-crash.ll new file mode 100644 index 0000000000000..683621a6f69bb --- /dev/null +++ b/test/Transforms/LoopVectorize/scev-exitlim-crash.ll @@ -0,0 +1,114 @@ +; RUN: opt < %s -loop-vectorize -force-vector-unroll=2 -force-vector-width=8 -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx" + +@b = common global i32 0, align 4 +@f = common global i32 0, align 4 +@a = common global i32 0, align 4 +@d = common global i32* null, align 8 +@e = common global i32* null, align 8 +@c = common global i32 0, align 4 + +; CHECK-LABEL: @fn1( +; CHECK: vector.body +define void @fn1() #0 { +entry: + br label %for.cond + +for.cond: ; preds = %for.cond, %entry + %i.0 = phi i32 [ undef, %entry ], [ %inc, %for.cond ] + %cmp = icmp slt i32 %i.0, 0 + %call = tail call i32 @fn2(double fadd (double fsub (double undef, double undef), double 1.000000e+00)) #2 + %inc = add nsw i32 %i.0, 1 + br i1 %cmp, label %for.cond, label %for.cond4.preheader + +for.cond4.preheader: ; preds = %for.cond + %call.lcssa = phi i32 [ %call, %for.cond ] + %cmp514 = icmp sgt i32 %call.lcssa, 0 + br i1 %cmp514, label %for.cond7.preheader.lr.ph, label %for.end26 + +for.cond7.preheader.lr.ph: ; preds = %for.cond4.preheader + %0 = load i32** @e, align 8, !tbaa !4 + br label %for.cond7.preheader + +for.cond7.preheader: ; preds = %for.cond7.preheader.lr.ph, %for.inc23 + %y.017 = phi i32 [ 0, %for.cond7.preheader.lr.ph ], [ %inc24, %for.inc23 ] + %i.116 = phi i32 [ 0, %for.cond7.preheader.lr.ph ], [ %i.2.lcssa, %for.inc23 ] + %n.015 = phi i32 [ undef, %for.cond7.preheader.lr.ph ], [ %inc25, %for.inc23 ] + %1 = load i32* @b, align 4, !tbaa !5 + %tobool11 = icmp eq i32 %1, 0 + br i1 %tobool11, label %for.inc23, label %for.body8.lr.ph + +for.body8.lr.ph: ; preds = %for.cond7.preheader + %add9 = add i32 %n.015, 1 + br label %for.body8 + +for.body8: ; preds = %for.body8.lr.ph, %for.inc19 + %indvars.iv19 = phi i64 [ 0, %for.body8.lr.ph ], [ %indvars.iv.next20, %for.inc19 ] + %i.213 = phi i32 [ %i.116, %for.body8.lr.ph ], [ 0, %for.inc19 ] + %2 = trunc i64 %indvars.iv19 to i32 + %add10 = add i32 %add9, %2 + store i32 %add10, i32* @f, align 4, !tbaa !5 + %idx.ext = sext i32 %add10 to i64 + %add.ptr = getelementptr inbounds i32* @a, i64 %idx.ext + %tobool129 = icmp eq i32 %i.213, 0 + br i1 %tobool129, label %for.inc19, label %for.body13.lr.ph + +for.body13.lr.ph: ; preds = %for.body8 + %3 = sext i32 %i.213 to i64 + br label %for.body13 + +for.body13: ; preds = %for.body13.lr.ph, %for.body13 + %indvars.iv = phi i64 [ %3, %for.body13.lr.ph ], [ %indvars.iv.next, %for.body13 ] + %add.ptr.sum = add i64 %idx.ext, %indvars.iv + %arrayidx = getelementptr inbounds i32* @a, i64 %add.ptr.sum + %4 = load i32* %arrayidx, align 4, !tbaa !5 + %arrayidx15 = getelementptr inbounds i32* %0, i64 %indvars.iv + store i32 %4, i32* %arrayidx15, align 4, !tbaa !5 + %indvars.iv.next = add i64 %indvars.iv, 1 + %5 = trunc i64 %indvars.iv.next to i32 + %tobool12 = icmp eq i32 %5, 0 + br i1 %tobool12, label %for.cond11.for.inc19_crit_edge, label %for.body13 + +for.cond11.for.inc19_crit_edge: ; preds = %for.body13 + br label %for.inc19 + +for.inc19: ; preds = %for.cond11.for.inc19_crit_edge, %for.body8 + %6 = load i32* @c, align 4, !tbaa !5 + %inc20 = add nsw i32 %6, 1 + store i32 %inc20, i32* @c, align 4, !tbaa !5 + %indvars.iv.next20 = add i64 %indvars.iv19, 1 + %7 = load i32* @b, align 4, !tbaa !5 + %tobool = icmp eq i32 %7, 0 + br i1 %tobool, label %for.cond7.for.inc23_crit_edge, label %for.body8 + +for.cond7.for.inc23_crit_edge: ; preds = %for.inc19 + %add.ptr.lcssa = phi i32* [ %add.ptr, %for.inc19 ] + store i32* %add.ptr.lcssa, i32** @d, align 8, !tbaa !4 + br label %for.inc23 + +for.inc23: ; preds = %for.cond7.for.inc23_crit_edge, %for.cond7.preheader + %i.2.lcssa = phi i32 [ 0, %for.cond7.for.inc23_crit_edge ], [ %i.116, %for.cond7.preheader ] + %inc24 = add nsw i32 %y.017, 1 + %inc25 = add nsw i32 %n.015, 1 + %exitcond = icmp ne i32 %inc24, %call.lcssa + br i1 %exitcond, label %for.cond7.preheader, label %for.cond4.for.end26_crit_edge + +for.cond4.for.end26_crit_edge: ; preds = %for.inc23 + br label %for.end26 + +for.end26: ; preds = %for.cond4.for.end26_crit_edge, %for.cond4.preheader + ret void +} +declare i32 @fn2(double) #1 + +attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf"="true" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!0 = metadata !{metadata !"int", metadata !1} +!1 = metadata !{metadata !"omnipotent char", metadata !2} +!2 = metadata !{metadata !"Simple C/C++ TBAA"} +!3 = metadata !{metadata !"double", metadata !1} +!4 = metadata !{metadata !0, metadata !0, i64 0} +!5 = metadata !{metadata !3, metadata !3, i64 0} diff --git a/test/Transforms/LoopVectorize/simple-unroll.ll b/test/Transforms/LoopVectorize/simple-unroll.ll index 7e2dd5fc0fcf4..83f35ffb609b7 100644 --- a/test/Transforms/LoopVectorize/simple-unroll.ll +++ b/test/Transforms/LoopVectorize/simple-unroll.ll @@ -9,7 +9,7 @@ target triple = "x86_64-apple-macosx10.8.0" ; for (i=0; i<n; i++){ ; a[i] += i; ; } -;CHECK: @inc +;CHECK-LABEL: @inc( ;CHECK: load <4 x i32> ;CHECK: load <4 x i32> ;CHECK: add nsw <4 x i32> diff --git a/test/Transforms/LoopVectorize/small-loop.ll b/test/Transforms/LoopVectorize/small-loop.ll index fa83dba3d3679..49ce5c539727a 100644 --- a/test/Transforms/LoopVectorize/small-loop.ll +++ b/test/Transforms/LoopVectorize/small-loop.ll @@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.8.0" @b = common global [2048 x i32] zeroinitializer, align 16 @c = common global [2048 x i32] zeroinitializer, align 16 -;CHECK: @example1 +;CHECK-LABEL: @example1( ;CHECK-NOT: load <4 x i32> ;CHECK: ret void define void @example1() nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/start-non-zero.ll b/test/Transforms/LoopVectorize/start-non-zero.ll index e8a089a981209..8f675afd80cbc 100644 --- a/test/Transforms/LoopVectorize/start-non-zero.ll +++ b/test/Transforms/LoopVectorize/start-non-zero.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @start_at_nonzero +;CHECK-LABEL: @start_at_nonzero( ;CHECK: mul nuw <4 x i32> ;CHECK: ret i32 define i32 @start_at_nonzero(i32* nocapture %a, i32 %start, i32 %end) nounwind uwtable ssp { diff --git a/test/Transforms/LoopVectorize/store-shuffle-bug.ll b/test/Transforms/LoopVectorize/store-shuffle-bug.ll new file mode 100644 index 0000000000000..0ec8010756d19 --- /dev/null +++ b/test/Transforms/LoopVectorize/store-shuffle-bug.ll @@ -0,0 +1,55 @@ +; RUN: opt -S -loop-vectorize -force-vector-unroll=1 -force-vector-width=4 -dce -instcombine < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +@uf = common global [100 x i32] zeroinitializer, align 16 +@xi = common global [100 x i32] zeroinitializer, align 16 +@q = common global [100 x i32] zeroinitializer, align 16 + +; PR16455 + + +; Due to a bug in the way we handled reverse induction stores we would generate +; a shuffle too many. + +define void @t() { +entry: + br label %for.body + +; CHECK-LABEL: @t( +; CHECK: vector.body: +; CHECK: load <4 x i32> +; CHECK: [[VAR1:%[a-zA-Z0-9]+]] = shufflevector +; CHECK: load <4 x i32> +; CHECK: [[VAR2:%[a-zA-Z0-9]+]] = shufflevector +; CHECK: [[VAR3:%[a-zA-Z0-9]+]] = add nsw <4 x i32> [[VAR2]], [[VAR1]] +; CHECK: [[VAR4:%[a-zA-Z0-9]+]] = shufflevector <4 x i32> [[VAR3]] +; CHECK: store <4 x i32> [[VAR4]] +; CHECK: load <4 x i32> +; CHECK: [[VAR5:%[a-zA-Z0-9]+]] = shufflevector +; CHECK-NOT: add nsw <4 x i32> [[VAR4]], [[VAR5]] +; CHECK-NOT: add nsw <4 x i32> [[VAR5]], [[VAR4]] +; CHECK: add nsw <4 x i32> [[VAR3]], [[VAR5]] + +for.body: + %indvars.iv = phi i64 [ 93, %entry ], [ %indvars.iv.next, %for.body ] + %0 = add i64 %indvars.iv, 1 + %arrayidx = getelementptr inbounds [100 x i32]* @uf, i64 0, i64 %0 + %arrayidx3 = getelementptr inbounds [100 x i32]* @xi, i64 0, i64 %0 + %1 = load i32* %arrayidx3, align 4 + %2 = load i32* %arrayidx, align 4 + %add4 = add nsw i32 %2, %1 + store i32 %add4, i32* %arrayidx, align 4 + %arrayidx7 = getelementptr inbounds [100 x i32]* @q, i64 0, i64 %0 + %3 = load i32* %arrayidx7, align 4 + %add8 = add nsw i32 %add4, %3 + store i32 %add8, i32* %arrayidx, align 4 + %indvars.iv.next = add i64 %indvars.iv, -1 + %4 = trunc i64 %indvars.iv.next to i32 + %cmp = icmp ugt i32 %4, 2 + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret void +} diff --git a/test/Transforms/LoopVectorize/struct_access.ll b/test/Transforms/LoopVectorize/struct_access.ll index 573480d77cdda..75beae82f170a 100644 --- a/test/Transforms/LoopVectorize/struct_access.ll +++ b/test/Transforms/LoopVectorize/struct_access.ll @@ -21,7 +21,7 @@ target triple = "x86_64-apple-macosx10.9.0" ; return sum; ; } -;CHECK: @foo +;CHECK-LABEL: @foo( ;CHECK-NOT: load <4 x i32> ;CHECK: ret define i32 @foo(%struct.coordinate* nocapture %A, i32 %n) nounwind uwtable readonly ssp { @@ -44,3 +44,45 @@ for.end: ; preds = %for.body, %entry %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] ret i32 %sum.0.lcssa } + +%struct.lit = type { i32 } + +; Verify that we still vectorize the access if the struct has the same size as +; the loaded element. +; struct lit { +; int x; +; }; +; +; +; int bar(struct lit *A, int n) { +; +; int sum = 0; +; for (int i = 0; i < n; ++i) +; sum += A[i].x; +; +; return sum; +; } + +;CHECK-LABEL: @bar( +;CHECK: load <4 x i32> +;CHECK: ret +define i32 @bar(%struct.lit* nocapture %A, i32 %n) nounwind uwtable readonly ssp { +entry: + %cmp4 = icmp sgt i32 %n, 0 + br i1 %cmp4, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %sum.05 = phi i32 [ %add, %for.body ], [ 0, %entry ] + %x = getelementptr inbounds %struct.lit* %A, i64 %indvars.iv, i32 0 + %0 = load i32* %x, align 4 + %add = add nsw i32 %0, %sum.05 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %for.body ] + ret i32 %sum.0.lcssa +} diff --git a/test/Transforms/LoopVectorize/undef-inst-bug.ll b/test/Transforms/LoopVectorize/undef-inst-bug.ll new file mode 100644 index 0000000000000..ed60e801afde3 --- /dev/null +++ b/test/Transforms/LoopVectorize/undef-inst-bug.ll @@ -0,0 +1,36 @@ +; RUN: opt -S -loop-vectorize -force-vector-width=4 -force-vector-unroll=1 < %s | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" + +; We use to fail on this loop because we did not properly handle the loop +; invariant instruction anchored in the loop when used as a getelementptr index. +; We would use the index from the original loop resulting in a use not dominated +; by the definition. + +; PR16452 + +; Verify that we don't miscompile this loop. + +; CHECK-LABEL: @t( +; CHECK: <4 x i32> + +define void @t() { +entry: + br label %for.body + +for.body: + %indvars.iv17 = phi i64 [ %indvars.next, %for.body ], [ 128, %entry ] + + ; Loop invariant anchored in loop. + %idxprom21 = zext i32 undef to i64 + + %arrayidx23 = getelementptr inbounds [100 x [100 x i32]]* undef, i64 0, i64 %idxprom21, i64 %indvars.iv17 + store i32 undef, i32* %arrayidx23, align 4 + %indvars.next= add i64 %indvars.iv17, -1 + %0 = trunc i64 %indvars.next to i32 + %cmp15 = icmp ugt i32 %0, undef + br i1 %cmp15, label %for.body, label %loopexit + +loopexit: + ret void +} diff --git a/test/Transforms/LoopVectorize/unroll_novec.ll b/test/Transforms/LoopVectorize/unroll_novec.ll new file mode 100644 index 0000000000000..33f128da905df --- /dev/null +++ b/test/Transforms/LoopVectorize/unroll_novec.ll @@ -0,0 +1,39 @@ +; RUN: opt < %s -loop-vectorize -force-vector-width=1 -force-vector-unroll=2 -dce -instcombine -S | FileCheck %s + +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" +target triple = "x86_64-apple-macosx10.8.0" + +@a = common global [2048 x i32] zeroinitializer, align 16 + +; This is the loop. +; for (i=0; i<n; i++){ +; a[i] += i; +; } +;CHECK-LABEL: @inc( +;CHECK: load i32* +;CHECK: load i32* +;CHECK: add nsw i32 +;CHECK: add nsw i32 +;CHECK: store i32 +;CHECK: store i32 +;CHECK: ret void +define void @inc(i32 %n) nounwind uwtable noinline ssp { + %1 = icmp sgt i32 %n, 0 + br i1 %1, label %.lr.ph, label %._crit_edge + +.lr.ph: ; preds = %0, %.lr.ph + %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ] + %2 = getelementptr inbounds [2048 x i32]* @a, i64 0, i64 %indvars.iv + %3 = load i32* %2, align 4 + %4 = trunc i64 %indvars.iv to i32 + %5 = add nsw i32 %3, %4 + store i32 %5, i32* %2, align 4 + %indvars.iv.next = add i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %._crit_edge, label %.lr.ph + +._crit_edge: ; preds = %.lr.ph, %0 + ret void +} + diff --git a/test/Transforms/LoopVectorize/value-ptr-bug.ll b/test/Transforms/LoopVectorize/value-ptr-bug.ll index f376656f0754d..e8d37285f8037 100644 --- a/test/Transforms/LoopVectorize/value-ptr-bug.ll +++ b/test/Transforms/LoopVectorize/value-ptr-bug.ll @@ -9,7 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; SCEVExpander::expandCodeFor would change a value (the start value of an ; induction) that we cached in the induction variable list. -; CHECK: test_vh +; CHECK-LABEL: @test_vh( ; CHECK-NOT: store <4 x i8> undef define void @test_vh(i32* %ptr265, i32* %ptr266, i32 %sub267) { diff --git a/test/Transforms/LoopVectorize/vectorize-once.ll b/test/Transforms/LoopVectorize/vectorize-once.ll index f289ded25de1a..780046930e1bb 100644 --- a/test/Transforms/LoopVectorize/vectorize-once.ll +++ b/test/Transforms/LoopVectorize/vectorize-once.ll @@ -9,9 +9,9 @@ target triple = "x86_64-apple-macosx10.8.0" ; This test checks that we add metadata to vectorized loops -; CHECK: _Z4foo1Pii +; CHECK-LABEL: @_Z4foo1Pii( ; CHECK: <4 x i32> -; CHECK: llvm.vectorizer.already_vectorized +; CHECK: llvm.loop ; CHECK: ret ; This test comes from the loop: @@ -40,10 +40,10 @@ _ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %for.body.i, %entry ret i32 %__init.addr.0.lcssa.i } -; This test checks that we don't vectorize loops that are marked with the "already vectorized" metadata. -; CHECK: _Z4foo2Pii +; This test checks that we don't vectorize loops that are marked with the "width" == 1 metadata. +; CHECK-LABEL: @_Z4foo2Pii( ; CHECK-NOT: <4 x i32> -; CHECK: llvm.vectorizer.already_vectorized +; CHECK: llvm.loop ; CHECK: ret define i32 @_Z4foo2Pii(i32* %A, i32 %n) #0 { entry: @@ -59,7 +59,7 @@ for.body.i: ; preds = %entry, %for.body.i %add.i = add nsw i32 %0, %__init.addr.05.i %incdec.ptr.i = getelementptr inbounds i32* %__first.addr.04.i, i64 1 %cmp.i = icmp eq i32* %incdec.ptr.i, %add.ptr - br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i, !llvm.vectorizer.already_vectorized !3 + br i1 %cmp.i, label %_ZSt10accumulateIPiiET0_T_S2_S1_.exit, label %for.body.i, !llvm.loop !0 _ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %for.body.i, %entry %__init.addr.0.lcssa.i = phi i32 [ 0, %entry ], [ %add.i, %for.body.i ] @@ -68,5 +68,10 @@ _ZSt10accumulateIPiiET0_T_S2_S1_.exit: ; preds = %for.body.i, %entry attributes #0 = { nounwind readonly ssp uwtable "fp-contract-model"="standard" "no-frame-pointer-elim" "no-frame-pointer-elim-non-leaf" "realign-stack" "relocation-model"="pic" "ssp-buffers-size"="8" } -!3 = metadata !{} +; CHECK: !0 = metadata !{metadata !0, metadata !1, metadata !2} +; CHECK: !1 = metadata !{metadata !"llvm.vectorizer.width", i32 1} +; CHECK: !2 = metadata !{metadata !"llvm.vectorizer.unroll", i32 1} +; CHECK: !3 = metadata !{metadata !3, metadata !1, metadata !2} +!0 = metadata !{metadata !0, metadata !1} +!1 = metadata !{metadata !"llvm.vectorizer.width", i32 1} diff --git a/test/Transforms/LoopVectorize/write-only.ll b/test/Transforms/LoopVectorize/write-only.ll index 54cbe8df46b06..71a9cd0dc5bec 100644 --- a/test/Transforms/LoopVectorize/write-only.ll +++ b/test/Transforms/LoopVectorize/write-only.ll @@ -3,7 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.8.0" -;CHECK: @read_mod_write_single_ptr +;CHECK-LABEL: @read_mod_write_single_ptr( ;CHECK: load <4 x float> ;CHECK: ret i32 define i32 @read_mod_write_single_ptr(float* nocapture %a, i32 %n) nounwind uwtable ssp { |